1 package com.acumenvelocity.ath.common;
2
3 import java.io.File;
4 import java.io.FileInputStream;
5 import java.io.IOException;
6 import java.io.InputStream;
7 import java.net.URI;
8 import java.nio.charset.StandardCharsets;
9 import java.nio.file.Files;
10 import java.nio.file.Paths;
11 import java.util.ArrayList;
12 import java.util.Collections;
13 import java.util.Comparator;
14 import java.util.Enumeration;
15 import java.util.Iterator;
16 import java.util.List;
17 import java.util.Map;
18 import java.util.ResourceBundle;
19 import java.util.TreeMap;
20 import java.util.UUID;
21 import java.util.concurrent.ConcurrentHashMap;
22 import java.util.concurrent.ConcurrentMap;
23
24 import org.apache.solr.client.solrj.response.QueryResponse;
25 import org.apache.solr.common.SolrDocument;
26 import org.apache.solr.common.SolrInputDocument;
27 import org.bson.types.ObjectId;
28
29 import com.acumenvelocity.ath.common.exception.AthRuntimeException;
30 import com.acumenvelocity.ath.controller.DocumentController;
31 import com.acumenvelocity.ath.filters.pdf.AthPdfFilter;
32 import com.acumenvelocity.ath.filters.pdf.AthPdfFilterWriter;
33 import com.acumenvelocity.ath.filters.pdf.Parameters;
34 import com.acumenvelocity.ath.gcs.AthStorage;
35 import com.acumenvelocity.ath.model.DocumentStatus;
36 import com.acumenvelocity.ath.model.EncodingInfo;
37 import com.acumenvelocity.ath.model.FilterInfo;
38 import com.acumenvelocity.ath.model.LanguageInfo;
39 import com.acumenvelocity.ath.model.MtResources;
40 import com.acumenvelocity.ath.model.OcrMode;
41 import com.acumenvelocity.ath.model.ParametersFormat;
42 import com.acumenvelocity.ath.pagematcher.DocxPageMatcher;
43 import com.acumenvelocity.ath.solr.AthIndex;
44 import com.acumenvelocity.ath.solr.doc.SolrDocMergerStep;
45 import com.acumenvelocity.ath.solr.doc.SolrDocWriterStep;
46 import com.acumenvelocity.ath.solr.tm.SolrTmConnector;
47 import com.acumenvelocity.ath.srx.SrxFileMapper;
48 import com.acumenvelocity.ath.steps.BatchMtParameters;
49 import com.acumenvelocity.ath.steps.BatchMtStep;
50 import com.acumenvelocity.ath.steps.CodesReinsertionStep;
51 import com.acumenvelocity.ath.steps.MtConfidenceScoringStep;
52 import com.acumenvelocity.ath.steps.MtLeveragingStep;
53 import com.acumenvelocity.ath.steps.SegmentTrimmerStep;
54 import com.fasterxml.jackson.databind.JsonNode;
55 import com.google.auth.oauth2.GoogleCredentials;
56 import com.google.auth.oauth2.ServiceAccountCredentials;
57
58 import io.swagger.oas.inflector.models.ResponseContext;
59 import net.sf.okapi.applications.rainbow.MainForm;
60 import net.sf.okapi.applications.rainbow.lib.EncodingItem;
61 import net.sf.okapi.applications.rainbow.lib.EncodingManager;
62 import net.sf.okapi.applications.rainbow.lib.LanguageItem;
63 import net.sf.okapi.applications.rainbow.lib.LanguageManager;
64 import net.sf.okapi.common.BOMNewlineEncodingDetector;
65 import net.sf.okapi.common.IParametersEditorMapper;
66 import net.sf.okapi.common.ListUtil;
67 import net.sf.okapi.common.LocaleId;
68 import net.sf.okapi.common.MimeTypeMapper;
69 import net.sf.okapi.common.StringUtil;
70 import net.sf.okapi.common.Util;
71 import net.sf.okapi.common.filters.FilterConfiguration;
72 import net.sf.okapi.common.filters.IFilter;
73 import net.sf.okapi.common.filters.IFilterConfigurationMapper;
74 import net.sf.okapi.common.filters.SharedFilterConfigurationMapper;
75 import net.sf.okapi.common.pipeline.IPipelineStep;
76 import net.sf.okapi.common.pipeline.PipelineReturnValue;
77 import net.sf.okapi.common.pipelinebuilder.XBatch;
78 import net.sf.okapi.common.pipelinebuilder.XBatchItem;
79 import net.sf.okapi.common.pipelinebuilder.XDocument;
80 import net.sf.okapi.common.pipelinebuilder.XParameter;
81 import net.sf.okapi.common.pipelinebuilder.XPipeline;
82 import net.sf.okapi.common.pipelinebuilder.XPipelineStep;
83 import net.sf.okapi.connectors.google.GoogleMTv2Connector;
84 import net.sf.okapi.connectors.google.GoogleMTv2Parameters;
85 import net.sf.okapi.filters.openxml.OpenXMLFilter;
86 import net.sf.okapi.steps.common.FilterEventsToRawDocumentStep;
87 import net.sf.okapi.steps.common.RawDocumentToFilterEventsStep;
88 import net.sf.okapi.steps.desegmentation.DesegmentationStep;
89 import net.sf.okapi.steps.heuristicaligner.HeuristicSentenceAlignerStep;
90 import net.sf.okapi.steps.leveraging.LeveragingStep;
91 import net.sf.okapi.steps.llmsentencealigner.LlmSentenceAlignerStep;
92 import net.sf.okapi.steps.segmentation.SegmentationStep;
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117 public class ControllerUtil {
118 private static final String BUNDLE_NAME = "net.sf.okapi.common.filters.DefaultFilters";
119 private static String projectId;
120
121 private static final List<LanguageInfo> languageInfos;
122 private static final List<EncodingInfo> encodingInfos;
123 private static final List<FilterInfo> filterInfos;
124
125 private static IFilterConfigurationMapper fcMapper = new SharedFilterConfigurationMapper();
126 private static Map<String, String> extensionsMap = new TreeMap<>();
127
128
129 static final ConcurrentMap<UUID, XPipeline> activePipelines = new ConcurrentHashMap<>();
130
131 static {
132
133
134
135 fcMapper.addConfigurations(AthPdfFilter.class.getName());
136
137 languageInfos = createLanguageInfos();
138 encodingInfos = createEncodingInfos();
139 filterInfos = createFilterInfos();
140
141 projectId = getProjectIdFromKeyFile();
142 }
143
144 public static List<LanguageInfo> getLanguageInfos() {
145 return languageInfos;
146 }
147
148 public static List<EncodingInfo> getEncodingInfos() {
149 return encodingInfos;
150 }
151
152 public static List<FilterInfo> getFilterInfos() {
153 return filterInfos;
154 }
155
156 public static IFilterConfigurationMapper getFcMapper() {
157 return fcMapper;
158 }
159
160 public static boolean checkObjectIdParam(String param) {
161 return checkParam(param) && ObjectId.isValid(param);
162 }
163
164
165
166
167
168
169
170
171
172
173 public static void setMappings(IFilterConfigurationMapper fcMapper, boolean reset,
174 boolean addConfigurations) {
175
176
177 ResourceBundle res = ResourceBundle.getBundle(BUNDLE_NAME);
178 Enumeration<String> keys = res.getKeys();
179 ArrayList<String> list = Collections.list(keys);
180
181 if (reset) {
182 fcMapper.clearConfigurations(false);
183 ((IParametersEditorMapper) fcMapper).clearDescriptionProviders();
184 ((IParametersEditorMapper) fcMapper).clearEditors();
185 }
186
187
188 for (String key : list) {
189
190 if (!key.startsWith("filterClass"))
191 continue;
192
193 try {
194 int n = key.indexOf('_');
195 String suffix = key.substring(n);
196 String value = res.getString(key);
197
198
199 if (addConfigurations) {
200 fcMapper.addConfigurations(value);
201 }
202
203 String key2 = "parametersClass" + suffix;
204
205 if (list.contains(key2)) {
206 String paramsClass = res.getString(key2);
207
208
209 String key3 = "parametersEditorClass" + suffix;
210
211 if (list.contains(key3)) {
212 value = res.getString(key3);
213 ((IParametersEditorMapper) fcMapper).addEditor(value, paramsClass);
214
215 } else {
216 key3 = "editorDescriptionProvider" + suffix;
217
218 if (list.contains(key3)) {
219 value = res.getString(key3);
220 ((IParametersEditorMapper) fcMapper).addDescriptionProvider(value, paramsClass);
221 }
222 }
223 }
224
225 } catch (Exception ex) {
226 Log.warn(ControllerUtil.class,
227 "Error while trying to build filter for the property key '{}' -- {}", key,
228 ex.getMessage());
229
230 continue;
231 }
232 }
233 }
234
235 private static List<FilterInfo> createFilterInfos() {
236 List<FilterInfo> filterInfos = new ArrayList<>();
237 extensionsMap.clear();
238
239 setMappings(fcMapper, false, true);
240 Iterator<FilterConfiguration> iter = fcMapper.getAllConfigurations();
241 StringBuilder sb = new StringBuilder();
242
243 while (iter.hasNext()) {
244 FilterConfiguration fc = iter.next();
245
246 try {
247 IFilter filter = fcMapper.createFilter(fc.configId);
248 sb.append(Log.format("{} -- {}\n", fc.configId, fc.name));
249
250 String paramsSt = filter == null || filter.getParameters() == null ? ""
251 : filter.getParameters().toString();
252
253 FilterInfo fi = new FilterInfo()
254 .id(fc.configId)
255 .name(fc.name)
256 .description(fc.description)
257 .custom(fc.custom)
258 .mimeType(fc.mimeType)
259 .fileExtensions(ListUtil.stringAsList(fc.extensions, ";"))
260 .parameters(paramsSt)
261 .parametersFormat(getFormat(paramsSt));
262
263 for (String ext : fi.getFileExtensions()) {
264 extensionsMap.put(ext, fi.getId());
265 }
266
267 filterInfos.add(fi);
268
269 } catch (Exception e) {
270 Log.warn(ControllerUtil.class, "Cannot instantiate the filter '{}' -- {}", fc.filterClass,
271 e.getMessage());
272
273 continue;
274 }
275 }
276
277 Collections.sort(filterInfos,
278 Comparator.comparing(FilterInfo::getName, String.CASE_INSENSITIVE_ORDER));
279
280 Log.debug(ControllerUtil.class, "\n\n------ Filter configs:\n{}\n\n", sb.toString());
281 return Collections.unmodifiableList(filterInfos);
282
283 }
284
285 private static ParametersFormat getFormat(String st) {
286 if (st.startsWith("#v")) {
287 return ParametersFormat.FPRM;
288
289 } else if (st.startsWith("<?xml")) {
290 return ParametersFormat.XML;
291
292 } else {
293 return ParametersFormat.YAML;
294 }
295 }
296
297 private static List<EncodingInfo> createEncodingInfos() {
298 List<EncodingInfo> encodings = new ArrayList<>();
299 EncodingManager em = new EncodingManager();
300
301 try (InputStream is = MainForm.class.getResourceAsStream("/shared/encodings.xml")) {
302 em.loadList(is);
303 EncodingItem ei;
304
305 for (int i = 0; i < em.getCount(); i++) {
306 ei = em.getItem(i);
307
308 EncodingInfo item = new EncodingInfo()
309 .name(ei.name)
310 .codePage(ei.codePage)
311 .ianaName(ei.ianaName);
312
313 encodings.add(item);
314 }
315
316 } catch (IOException e) {
317 Log.warn(ControllerUtil.class, "Error getting encodings -- {}", e.getMessage());
318 }
319
320 return Collections.unmodifiableList(encodings);
321 }
322
323 private static List<LanguageInfo> createLanguageInfos() {
324 List<LanguageInfo> langInfos = new ArrayList<>();
325
326 LanguageManager lm = new LanguageManager();
327 LanguageItem li;
328
329 for (int i = 0; i < lm.getCount(); i++) {
330 li = lm.getItem(i);
331
332 LanguageInfo item = new LanguageInfo();
333
334 item.setName(li.name);
335 item.setIsoCode(li.code);
336
337 langInfos.add(item);
338 }
339
340 return Collections.unmodifiableList(langInfos);
341 }
342
343 public static boolean checkParam(String param) {
344
345
346 return !Util.isEmpty(param) && !"null".equalsIgnoreCase(param);
347 }
348
349 public static boolean checkParam(Object param) {
350 return param != null;
351 }
352
353
354
355
356
357
358
359
360
361
362 public static <T> List<T> getPage(List<T> sourceList, int page, int pageSize) {
363 if (pageSize <= 0 || page <= 0) {
364 throw new IllegalArgumentException("Invalid page size: " + pageSize);
365 }
366
367 int fromIndex = (page - 1) * pageSize;
368
369 if (sourceList == null || sourceList.size() <= fromIndex) {
370 return Collections.emptyList();
371 }
372
373
374 return sourceList.subList(fromIndex, Math.min(fromIndex + pageSize, sourceList.size()));
375 }
376
377 public static String getFilterId(String fileExt) {
378 return extensionsMap.get(fileExt);
379 }
380
381 public static boolean checkUuidParam(String param) {
382 return checkParam(param) && isValidUuid(param);
383 }
384
385 private static boolean isValidUuid(String uuid) {
386
387 String uuidPattern = "^[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?"
388 + "[0-9a-fA-F]{12}$";
389
390 return uuid.matches(uuidPattern);
391 }
392
393 public static ResponseContext importFile(
394 SolrInputDocument doc,
395 UUID docId,
396 String docFileName,
397 URI docGcsUrl,
398 String docEncoding,
399 String srcLang,
400 String trgLang,
401 String filterId,
402 String filterParams,
403 String srcSrx,
404 UUID tmId,
405 Integer tmThreshold,
406 String mtEngineId,
407 String mtEngineParams,
408 List<MtResources> mtCustomResources,
409 boolean mtProvideConfidenceScores,
410 boolean mtUseTranslateLlm,
411 boolean mtSendPlainText,
412 boolean useCodesReinsertionModel,
413 String codesReinsertionModelName,
414 UUID userId,
415 boolean newDoc) throws Exception {
416
417 IFilterConfigurationMapper fcMapper = ControllerUtil.getFcMapper();
418
419 if (Util.isEmpty(filterId)) {
420
421 String fileExt = Util.getExtension(docFileName);
422
423 if (Util.isEmpty(fileExt)) {
424 throw new AthRuntimeException("No file extension or filterId specified");
425
426 } else {
427 filterId = ControllerUtil.getFilterId(fileExt);
428
429 if (Util.isEmpty(filterId)) {
430 throw new AthRuntimeException("Unknown file extension");
431 }
432 }
433 }
434
435 try (IFilter filter = fcMapper.createFilter(filterId)) {
436 if (!Util.isEmpty(filterParams)) {
437
438 filter.getParameters().fromString(filterParams);
439 }
440
441 doc.setField(Const.ATH_PROP_FILTER_ID, filterId);
442
443 if (filterParams == null && filter.getParameters() != null) {
444 filterParams = filter.getParameters().toString();
445 }
446
447 if (filterParams != null) {
448 doc.setField(Const.ATH_PROP_FILTER_PARAMS, filterParams);
449 }
450
451 Log.info(ControllerUtil.class, "{} (id='{}') was created", filter.getName(), filterId);
452
453 if (Util.isEmpty(srcSrx)) {
454 srcSrx = getSrx(srcLang);
455 }
456
457 doc.setField(Const.ATH_PROP_SRC_SRX, srcSrx);
458
459 LeveragingStep ls = tmId == null ? null : new LeveragingStep();
460
461 if (ls != null) {
462 net.sf.okapi.steps.leveraging.Parameters lsParams = ls.getParameters();
463 lsParams.setResourceClassName(SolrTmConnector.class.getName());
464
465 com.acumenvelocity.ath.solr.tm.Parameters connParams = new com.acumenvelocity.ath.solr.tm.Parameters();
466 connParams.setTmId(tmId.toString());
467
468 lsParams.setResourceParameters(connParams.toString());
469 lsParams.setThreshold(tmThreshold);
470 lsParams.setFillTarget(true);
471 lsParams.setFillTargetThreshold(tmThreshold);
472 }
473
474 boolean useMt = !Util.isEmpty(mtEngineId);
475 IPipelineStep mts = null;
476
477 if (useMt) {
478 switch (mtEngineId) {
479 case Const.MT_PROVIDER_GOOGLE_MT:
480 MtLeveragingStep v2Mts = new MtLeveragingStep(mtSendPlainText);
481 mts = v2Mts;
482
483 net.sf.okapi.steps.leveraging.Parameters lsParams = v2Mts.getParameters();
484 lsParams.setResourceClassName(GoogleMTv2Connector.class.getName());
485
486 if (Util.isEmpty(mtEngineParams)) {
487 GoogleMTv2Parameters connParams = new GoogleMTv2Parameters();
488 connParams.setApiKey(Const.ATH_GCT_API_KEY);
489 connParams.setRetryIntervalMs(0);
490 connParams.setRetryCount(1);
491
492 lsParams.setResourceParameters(connParams.toString());
493
494 } else {
495 lsParams.setResourceParameters(mtEngineParams);
496 }
497
498 break;
499
500 case Const.MT_PROVIDER_GOOGLE_MT_V3:
501 BatchMtStep bmts = new BatchMtStep();
502 mts = bmts;
503
504 BatchMtParameters bmtsParams = bmts.getParameters();
505
506 bmtsParams.setApiKey(Const.ATH_GCT_API_KEY);
507 bmtsParams.setProjectLocation(Const.ATH_GCP_PROJECT_LOCATION);
508 bmtsParams.setCredentialsPath(Const.ATH_GCP_SECRET_FILE);
509
510 boolean isCustomModel = mtCustomResources != null && mtCustomResources.size() > 0;
511
512 if (isCustomModel) {
513 MtResources res = mtCustomResources.get(0);
514 bmtsParams.setGlossaryId(res.getMtGlossaryId());
515 bmtsParams.setModelId(res.getMtModelId());
516 }
517
518 bmtsParams.setMimeType(mtSendPlainText ? MimeTypeMapper.PLAIN_TEXT_MIME_TYPE
519 : MimeTypeMapper.HTML_MIME_TYPE);
520
521 bmtsParams.setRetryIntervalMs(0);
522 bmtsParams.setRetryCount(1);
523 bmtsParams.setFailuresBeforeAbort(-1);
524
525 bmtsParams.setMtUseTranslateLlm(mtUseTranslateLlm);
526 bmtsParams.setMtSendPlainText(mtSendPlainText);
527 break;
528
529 default:
530 break;
531 }
532 }
533
534
535
536
537
538
539
540
541
542 if (filter instanceof AthPdfFilter) {
543 Parameters params = (Parameters) filter.getParameters();
544
545 if (params != null && params.getOcrMode() == OcrMode.AUTO) {
546 File docFile = AthUtil.createTempFile();
547
548 try {
549 AthStorage.storeFile(docGcsUrl, MimeTypeMapper.PDF_MIME_TYPE, docFile);
550 boolean needsOcr = PdfUtil.needsOcr(docFile);
551
552 if (needsOcr) {
553 params.setOcrMode(OcrMode.ENABLED);
554
555 } else {
556 params.setOcrMode(OcrMode.DISABLED);
557 }
558
559 } catch (Exception e) {
560 Log.error(ControllerUtil.class, e, "PDF conversion error");
561
562 } finally {
563 try {
564 Files.deleteIfExists(docFile.toPath());
565
566 } catch (Exception ignored) {
567 }
568 }
569 }
570 }
571
572
573 try (InputStream is = AthStorage.getInputStream(docGcsUrl);
574
575 XPipeline pl = new XPipeline(
576 "Import pipeline",
577
578 new XBatch(
579 new XBatchItem(
580 new XDocument(
581 is,
582 docEncoding,
583 LocaleId.fromString(srcLang),
584 LocaleId.fromString(trgLang)))),
585
586 new RawDocumentToFilterEventsStep(filter),
587
588 new XPipelineStep(
589 new SegmentationStep(),
590 new XParameter("sourceSrx", srcSrx),
591 new XParameter("targetSrx", null),
592 new XParameter("segmentSource", !Util.isEmpty(srcSrx)),
593 new XParameter("segmentTarget", false),
594 new XParameter("copySource", false)),
595
596 new SegmentTrimmerStep(),
597 ls,
598
599 mtProvideConfidenceScores
600 ? new MtConfidenceScoringStep(mtCustomResources, mtSendPlainText)
601 : mts,
602
603 mtSendPlainText ?
604
605
606 new CodesReinsertionStep(useCodesReinsertionModel, codesReinsertionModelName)
607 : null,
608
609 new SolrDocWriterStep(docId, docFileName, userId, newDoc))) {
610
611 PipelineReturnValue res = executePipelineWithCancellationCheck(pl, docId);
612
613 if (res == PipelineReturnValue.SUCCEDED) {
614 return Response.success(200);
615
616 } else if (res == PipelineReturnValue.CANCELLED) {
617 return Response.error(499, "Import cancelled");
618 }
619 }
620
621 return Response.error(500, "Import failed");
622
623 } catch (Exception e) {
624 String st = newDoc ? "Import failed" : "Update failed";
625 Log.error(ControllerUtil.class, e, st);
626 return Response.error(500, e, st);
627 }
628 }
629
630 public static ResponseContext alignFile(
631 SolrInputDocument doc,
632 UUID docId,
633 File docFile,
634 String docFileName,
635 URI docGcsUrl,
636 String docEncoding,
637 URI docTrlGcsUrl,
638 String docTrlEncoding,
639 String srcLang,
640 String trgLang,
641 String filterId,
642 String filterParams,
643 String srcSrx,
644 String trgSrx,
645 boolean mtSendPlainText,
646 boolean useAlignmentModel,
647 String alignmentModelName,
648 boolean useCodesReinsertionModel,
649 String codesReinsertionModelName,
650 UUID userId) throws Exception {
651
652 boolean alignWithTranslation = docTrlGcsUrl != null;
653 IFilterConfigurationMapper fcMapper = ControllerUtil.getFcMapper();
654
655 if (Util.isEmpty(filterId)) {
656
657 String fileExt = Util.getExtension(docFileName);
658
659 if (Util.isEmpty(fileExt)) {
660 throw new AthRuntimeException("No file extension or filterId specified");
661
662 } else {
663 filterId = ControllerUtil.getFilterId(fileExt);
664
665 if (Util.isEmpty(filterId)) {
666 throw new AthRuntimeException("Unknown file extension");
667 }
668 }
669 }
670
671 try (IFilter filter = fcMapper.createFilter(filterId)) {
672 if (!Util.isEmpty(filterParams)) {
673
674 filter.getParameters().fromString(filterParams);
675 }
676
677 doc.setField(Const.ATH_PROP_FILTER_ID, filterId);
678
679 if (filterParams == null && filter.getParameters() != null) {
680 filterParams = filter.getParameters().toString();
681 }
682
683 if (filterParams != null) {
684 doc.setField(Const.ATH_PROP_FILTER_PARAMS, filterParams);
685 }
686
687 Log.info(ControllerUtil.class, "{} (id='{}') was created", filter.getName(), filterId);
688
689 if (Util.isEmpty(srcSrx)) {
690 srcSrx = getSrx(srcLang);
691 }
692
693 doc.setField(Const.ATH_PROP_SRC_SRX, srcSrx);
694
695 if (Util.isEmpty(trgSrx)) {
696 if (alignWithTranslation) {
697 trgSrx = getSrx(trgLang);
698
699 } else {
700
701 }
702 }
703
704 if (!Util.isEmpty(trgSrx)) {
705 doc.setField(Const.ATH_PROP_TRG_SRX, trgSrx);
706 }
707
708
709
710 if (filter instanceof AthPdfFilter && docFile != null) {
711 Parameters params = (Parameters) filter.getParameters();
712
713 if (params != null && params.getOcrMode() == OcrMode.AUTO) {
714 boolean needsOcr = PdfUtil.needsOcr(docFile);
715
716 if (needsOcr) {
717 params.setOcrMode(OcrMode.ENABLED);
718
719 } else {
720 params.setOcrMode(OcrMode.DISABLED);
721 }
722 }
723 }
724
725 if (alignWithTranslation) {
726
727 File srcSrxFile = AthUtil.createTempFile();
728 File trgSrxFile = AthUtil.createTempFile();
729
730 try {
731
732 StringUtil.writeString(srcSrx, srcSrxFile);
733 StringUtil.writeString(trgSrx, trgSrxFile);
734
735 try (InputStream sis = AthStorage.getInputStream(docGcsUrl);
736 InputStream tis = AthStorage.getInputStream(docTrlGcsUrl);
737 IFilter targetFilter = fcMapper.createFilter(filterId);
738
739 XPipeline pl = new XPipeline(
740 "Alignment pipeline",
741
742 new XBatch(
743 new XBatchItem(
744 new XDocument(
745 sis,
746 docEncoding,
747 LocaleId.fromString(srcLang),
748 LocaleId.fromString(trgLang)),
749
750 new XDocument(
751 tis,
752 docTrlEncoding,
753 LocaleId.fromString(trgLang),
754 LocaleId.fromString(trgLang)))),
755
756 new RawDocumentToFilterEventsStep(filter),
757
758 !useAlignmentModel
759 ? new XPipelineStep(
760 new HeuristicSentenceAlignerStep(targetFilter),
761 new XParameter("generateTMX", false),
762 new XParameter("tmxOutputPath", null),
763 new XParameter("collapseWhitespace", false),
764 new XParameter("outputOneTOneMatchesOnly", false),
765 new XParameter("forceSimpleOneToOneAlignment", false),
766 new XParameter("segmentSource", true),
767 new XParameter("useCustomSourceRules", true),
768 new XParameter("customSourceRulesPath", srcSrxFile.getAbsolutePath()),
769 new XParameter("segmentTarget", true),
770 new XParameter("useCustomTargetRules", true),
771 new XParameter("customTargetRulesPath", trgSrxFile.getAbsolutePath()))
772
773 : new XPipelineStep(
774 new LlmSentenceAlignerStep(targetFilter),
775 new XParameter("useLlmAlignment", true),
776 new XParameter("llmModel", alignmentModelName),
777 new XParameter("useCodesReinsertionModel", useCodesReinsertionModel),
778 new XParameter("maxParagraphsPerRequest", 1000),
779 new XParameter("logAlignmentDetails", true),
780 new XParameter("collapseWhitespace", true),
781
782 new XParameter("segmentSource", !Util.isEmpty(srcSrx)),
783 new XParameter("useCustomSourceRules", !Util.isEmpty(srcSrx)),
784 new XParameter("customSourceRulesPath", srcSrxFile.getAbsolutePath()),
785
786 new XParameter("segmentTarget", !Util.isEmpty(trgSrx)),
787 new XParameter("useCustomTargetRules", !Util.isEmpty(trgSrx)),
788 new XParameter("customTargetRulesPath", trgSrxFile.getAbsolutePath())),
789
790
791
792
793
794
795 new CodesReinsertionStep(useCodesReinsertionModel, codesReinsertionModelName),
796 new SolrDocWriterStep(docId, docFileName, userId, false))) {
797
798
799
800
801
802 PipelineReturnValue res = executePipelineWithCancellationCheck(pl, docId);
803
804 if (res == PipelineReturnValue.SUCCEDED) {
805 return Response.success(200);
806
807 } else if (res == PipelineReturnValue.CANCELLED) {
808 return Response.error(499, "Alignment cancelled");
809 }
810 }
811
812 } finally {
813 if (srcSrxFile != null) {
814 srcSrxFile.delete();
815 }
816
817 if (trgSrxFile != null) {
818 trgSrxFile.delete();
819 }
820 }
821 }
822
823 return Response.error(500, "Alignment failed");
824
825 } catch (Exception e) {
826 String st = "Alignment failed";
827 Log.error(ControllerUtil.class, e, st);
828 return Response.error(500, e, st);
829 }
830 }
831
832
833
834
835
836
837
838 private static String getSrx(String lang) {
839 String fn = SrxFileMapper.getSrxFileName(lang);
840
841 String srxFileName = (fn == null)
842 ? "/srx/alternate-default.srx"
843
844 : "/srx/memoq/default/v9.0/" + fn;
845
846 try (InputStream in = ControllerUtil.class.getResourceAsStream(srxFileName)) {
847 if (in == null) {
848 throw new IllegalArgumentException("SRX file not found: " + srxFileName);
849 }
850
851
852 BOMNewlineEncodingDetector detector = new BOMNewlineEncodingDetector(in,
853 StandardCharsets.UTF_8);
854
855 detector.detectAndRemoveBom();
856
857
858 byte[] data = detector.getInputStream().readAllBytes();
859 return new String(data, StandardCharsets.UTF_8);
860
861 } catch (Exception e) {
862 throw new RuntimeException("Failed to read SRX file: " + srxFileName, e);
863 }
864 }
865
866 public static ResponseContext exportFile(
867 SolrDocument doc,
868 SolrDocument tmDoc,
869 SolrInputDocument updateDoc,
870 URI docOutGcsUrl,
871 String docOutEncoding,
872 UUID tmId,
873 boolean adjustTargetPages,
874 UUID userId) throws Exception {
875
876 String catFrameworkName = SolrUtil.safeGetField(doc, Const.ATH_PROP_CAT_FRAMEWORK_NAME,
877 null);
878
879 String catFrameworkVersion = SolrUtil.safeGetField(doc, Const.ATH_PROP_CAT_FRAMEWORK_VERSION,
880 null);
881
882
883
884
885
886
887
888
889
890
891
892
893
894 if (catFrameworkName != null && !catFrameworkName.equals(Const.CAT_FRAMEWORK_NAME)) {
895 throw new AthRuntimeException("CAT framework mismatch -- expected: {}, actual: {}",
896 catFrameworkName, Const.CAT_FRAMEWORK_NAME);
897 }
898
899 if (catFrameworkVersion != null && !catFrameworkVersion.equals(Const.CAT_FRAMEWORK_VERSION)) {
900 Log.warn(ControllerUtil.class,
901 "Export success is not guaranteed because of the {} version mismatch -- expected: {}, "
902 + "actual: {}",
903 Const.CAT_FRAMEWORK_NAME, catFrameworkVersion, Const.CAT_FRAMEWORK_VERSION);
904 }
905
906 IFilterConfigurationMapper fcMapper = ControllerUtil.getFcMapper();
907
908 UUID docId = AthUtil.safeToUuid(
909 SolrUtil.safeGetField(doc, Const.ATH_PROP_DOC_ID, null), null);
910
911 String filterId = SolrUtil.safeGetField(doc, Const.ATH_PROP_FILTER_ID, null);
912 String filterParams = SolrUtil.safeGetField(doc, Const.ATH_PROP_FILTER_PARAMS, null);
913 String docStorageName = SolrUtil.safeGetField(doc, Const.ATH_PROP_DOC_STORAGE_NAME, null);
914
915 String docFileEncoding = SolrUtil.safeGetField(doc, Const.ATH_PROP_DOC_FILE_ENCODING,
916 null);
917
918 String srcLang = SolrUtil.safeGetField(doc, Const.ATH_PROP_SRC_LANG, null);
919 String trgLang = SolrUtil.safeGetField(doc, Const.ATH_PROP_TRG_LANG, null);
920
921
922
923 IFilter filter = fcMapper.createFilter(filterId);
924
925
926 filter.getParameters().fromString(filterParams);
927
928 Log.info(ControllerUtil.class, "{} (id='{}') was created", filter.getName(), filterId);
929
930 File outFile = null;
931
932 try {
933 outFile = AthUtil.createTempFile();
934 boolean isPdf = filter instanceof AthPdfFilter;
935 boolean isDocx = filter instanceof OpenXMLFilter;
936
937 try (InputStream is = AthStorage.getInputStream(AthUtil.toURI(docStorageName));
938
939
940 InputStream is2 = isPdf || isDocx
941 ? AthStorage.getInputStream(AthUtil.toURI(docStorageName))
942 : null;
943
944 XPipeline pl = new XPipeline(
945 "Export pipeline",
946
947 new XBatch(
948 new XBatchItem(
949 new XDocument(
950 is,
951 docFileEncoding,
952 outFile.getAbsolutePath(),
953 docOutEncoding,
954 LocaleId.fromString(srcLang),
955 LocaleId.fromString(trgLang)))),
956
957 new RawDocumentToFilterEventsStep(filter),
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973 new SolrDocMergerStep(docId.toString()),
974 new DesegmentationStep(),
975
976 new FilterEventsToRawDocumentStep())) {
977
978 if (adjustTargetPages && filter instanceof AthPdfFilter) {
979
980
981 AthPdfFilter f = (AthPdfFilter) filter;
982 AthPdfFilterWriter fw = f.getFilterWriter();
983
984 fw.setDocxPostProcessor(docxFile -> {
985
986 try {
987 return DocxPageMatcher.matchPageCount(is2, docxFile, filter);
988
989 } catch (Exception e) {
990 AthRuntimeException.logAndThrow(ControllerUtil.class, e,
991 "Error matching PDF translation page count");
992 return null;
993 }
994 });
995 }
996
997 PipelineReturnValue res = executePipelineWithCancellationCheck(pl, docId);
998
999 if (res == PipelineReturnValue.SUCCEDED) {
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015 AthStorage.storeFile(docOutGcsUrl, filter.getMimeType(), outFile);
1016
1017 return Response.success(200);
1018
1019 } else if (res == PipelineReturnValue.CANCELLED) {
1020 return Response.error(499, "Export cancelled");
1021 }
1022 }
1023
1024 return Response.error(500, "Export failed");
1025
1026 } catch (Exception e) {
1027 Log.error(ControllerUtil.class, e, "Export failed");
1028 return Response.error(500, e, "Export failed");
1029 }
1030 }
1031
1032 public static String getProjectId() {
1033 return projectId;
1034 }
1035
1036
1037
1038
1039
1040 private static String getProjectIdFromKeyFile() {
1041 try (FileInputStream fis = new FileInputStream(Const.ATH_GCP_SECRET_FILE)) {
1042 GoogleCredentials credentials = GoogleCredentials.fromStream(fis);
1043
1044 if (credentials instanceof ServiceAccountCredentials) {
1045 ServiceAccountCredentials sac = (ServiceAccountCredentials) credentials;
1046 String projectId = sac.getProjectId();
1047
1048 if (projectId != null && !projectId.isBlank()) {
1049 return projectId;
1050 }
1051 }
1052
1053 } catch (Exception e) {
1054
1055
1056 Log.warn(ConversionUtil.class,
1057 "Official credential parsing failed ({}), falling back to JSON parsing",
1058 e.getMessage());
1059 }
1060
1061
1062 try {
1063 String json = Files.readString(Paths.get(Const.ATH_GCP_SECRET_FILE));
1064 JsonNode jsonNode = JacksonUtil.makeNode(json);
1065 return jsonNode.get(json) == null ? null : jsonNode.get(json).asText();
1066
1067 } catch (Exception ex) {
1068 Log.error(ConversionUtil.class, ex, "Failed to extract project_id from key file");
1069 }
1070
1071 return null;
1072 }
1073
1074 public static ResponseContext processFileImportPhase(
1075 SolrInputDocument doc,
1076 UUID docId,
1077 String docFileName,
1078 URI docGcsUrl,
1079 String docEncoding,
1080 String srcLang,
1081 String trgLang,
1082 String filterId,
1083 String filterParams,
1084 String srcSrx,
1085 UUID tmId,
1086 Integer tmThreshold,
1087 String mtEngineId,
1088 String mtEngineParams,
1089 List<MtResources> mtCustomResources,
1090 boolean mtProvideConfidenceScores,
1091 boolean mtUseTranslateLlm,
1092 boolean mtSendPlainText,
1093 boolean useCodesReinsertionModel,
1094 String codesReinsertionModelName,
1095 UUID userId,
1096 boolean newDoc) throws Exception {
1097
1098
1099 return importFile(doc, docId, docFileName, docGcsUrl, docEncoding,
1100 srcLang, trgLang, filterId, filterParams, srcSrx,
1101 tmId, tmThreshold, mtEngineId, mtEngineParams, mtCustomResources,
1102 mtProvideConfidenceScores, mtUseTranslateLlm, mtSendPlainText,
1103 useCodesReinsertionModel, codesReinsertionModelName, userId, newDoc);
1104 }
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114 private static PipelineReturnValue executePipelineWithCancellationCheck(
1115 XPipeline pipeline, UUID docId) {
1116
1117
1118 Thread cancellationMonitor = new Thread(() -> {
1119 activePipelines.put(docId, pipeline);
1120
1121 try {
1122 while (pipeline.getState() == PipelineReturnValue.RUNNING) {
1123 Thread.sleep(1000);
1124
1125 if (isCancelled(docId)) {
1126 Log.info(ControllerUtil.class, "Cancellation detected for docId: {}, stopping pipeline",
1127 docId);
1128
1129 pipeline.stop();
1130 break;
1131 }
1132 }
1133
1134 } catch (InterruptedException e) {
1135 Thread.currentThread().interrupt();
1136
1137 } finally {
1138
1139 activePipelines.remove(docId);
1140 }
1141 });
1142
1143 cancellationMonitor.setDaemon(true);
1144 cancellationMonitor.start();
1145
1146
1147 PipelineReturnValue result = pipeline.execute();
1148
1149
1150 cancellationMonitor.interrupt();
1151
1152 return result;
1153 }
1154
1155
1156
1157
1158
1159
1160
1161
1162 public static boolean isCancelled(UUID docId) {
1163 try {
1164 String query = Log.format("docId:\"{}\"", docId);
1165
1166 QueryResponse response = AthIndex.getMany(Const.SOLR_CORE_ATH_DOCS, query, null,
1167 QueryResponse.class);
1168
1169 if (response.getResults().isEmpty()) {
1170 return false;
1171 }
1172
1173 SolrDocument doc = response.getResults().get(0);
1174 String status = SolrUtil.safeGetField(doc, Const.ATH_PROP_STATUS, null);
1175
1176 return DocumentStatus.CANCELLED.toString().equals(status);
1177
1178 } catch (Exception e) {
1179 Log.error(DocumentController.class, e, "Error checking cancellation status for docId: {}",
1180 docId);
1181
1182 return false;
1183 }
1184 }
1185
1186 public static void stopPipeline(UUID docId) {
1187
1188 XPipeline pipeline = activePipelines.remove(docId);
1189
1190 if (pipeline != null) {
1191 try {
1192 pipeline.stop();
1193 Log.info(ControllerUtil.class, "Stopped active pipeline for docId: {}", docId);
1194
1195 } catch (Exception e) {
1196 Log.warn(ControllerUtil.class, "Error stopping pipeline for docId: {} -- {}", docId,
1197 e.getMessage());
1198 }
1199 }
1200 }
1201 }