1 package com.acumenvelocity.ath.controller;
2
3 import java.io.File;
4 import java.io.IOException;
5 import java.io.InputStream;
6 import java.net.URI;
7 import java.nio.file.Files;
8 import java.util.ArrayList;
9 import java.util.Date;
10 import java.util.List;
11 import java.util.Objects;
12 import java.util.UUID;
13 import java.util.concurrent.ExecutorService;
14 import java.util.concurrent.Executors;
15 import java.util.concurrent.TimeUnit;
16 import java.util.stream.Collectors;
17
18 import javax.ws.rs.core.Response.Status;
19
20 import org.apache.commons.io.FileUtils;
21 import org.apache.http.client.methods.HttpGet;
22 import org.apache.http.impl.client.CloseableHttpClient;
23 import org.apache.http.impl.client.HttpClients;
24 import org.apache.solr.client.solrj.SolrClient;
25 import org.apache.solr.client.solrj.SolrQuery;
26 import org.apache.solr.client.solrj.response.QueryResponse;
27 import org.apache.solr.common.SolrDocument;
28 import org.apache.solr.common.SolrDocumentList;
29 import org.apache.solr.common.SolrInputDocument;
30
31 import com.acumenvelocity.ath.common.AthUtil;
32 import com.acumenvelocity.ath.common.Const;
33 import com.acumenvelocity.ath.common.ControllerUtil;
34 import com.acumenvelocity.ath.common.JacksonUtil;
35 import com.acumenvelocity.ath.common.Log;
36 import com.acumenvelocity.ath.common.OkapiUtil;
37 import com.acumenvelocity.ath.common.Response;
38 import com.acumenvelocity.ath.common.SolrUtil;
39 import com.acumenvelocity.ath.common.exception.AthException;
40 import com.acumenvelocity.ath.gcs.AthStorage;
41 import com.acumenvelocity.ath.model.AlignDocumentRequest;
42 import com.acumenvelocity.ath.model.CancelledStatusResponse;
43 import com.acumenvelocity.ath.model.ConflictDownloadResponse;
44 import com.acumenvelocity.ath.model.ConflictResponse;
45 import com.acumenvelocity.ath.model.CreateDocumentSegmentRequest;
46 import com.acumenvelocity.ath.model.DocumentInfo;
47 import com.acumenvelocity.ath.model.DocumentInfosWrapper;
48 import com.acumenvelocity.ath.model.DocumentSegment;
49 import com.acumenvelocity.ath.model.DocumentSegmentWrapper;
50 import com.acumenvelocity.ath.model.DocumentSegmentsWrapper;
51 import com.acumenvelocity.ath.model.DocumentStatus;
52 import com.acumenvelocity.ath.model.ExportCompletedStatusResponse;
53 import com.acumenvelocity.ath.model.ExportDocumentRequest;
54 import com.acumenvelocity.ath.model.FailedStatusResponse;
55 import com.acumenvelocity.ath.model.ImportCompletedStatusResponse;
56 import com.acumenvelocity.ath.model.ImportDocumentRequest;
57 import com.acumenvelocity.ath.model.ModifiedSegmentsWrapper;
58 import com.acumenvelocity.ath.model.MtResources;
59 import com.acumenvelocity.ath.model.MtTargetInfo;
60 import com.acumenvelocity.ath.model.Origin;
61 import com.acumenvelocity.ath.model.PaginationInfo;
62 import com.acumenvelocity.ath.model.ProcessDocumentRequest;
63 import com.acumenvelocity.ath.model.ProcessingResponse;
64 import com.acumenvelocity.ath.model.ProcessingStatusResponse;
65 import com.acumenvelocity.ath.model.UpdateDocumentSegmentRequest;
66 import com.acumenvelocity.ath.model.x.LayeredTextX;
67 import com.acumenvelocity.ath.solr.AthIndex;
68 import com.fasterxml.jackson.core.type.TypeReference;
69 import com.fasterxml.jackson.databind.JsonNode;
70
71 import io.swagger.oas.inflector.models.RequestContext;
72 import io.swagger.oas.inflector.models.ResponseContext;
73 import net.sf.okapi.common.Util;
74
75 public class DocumentController {
76 private static final ExecutorService EXECUTOR = Executors.newFixedThreadPool(10);
77
78 private File fileToImport;
79 private boolean isTempFile;
80
81 static {
82 Runtime.getRuntime().addShutdownHook(new Thread(() -> {
83 shutdownExecutor();
84 }));
85 }
86
87
88 public static void shutdownExecutor() {
89 if (EXECUTOR != null && !EXECUTOR.isShutdown()) {
90 EXECUTOR.shutdown();
91
92 try {
93 if (!EXECUTOR.awaitTermination(60, TimeUnit.SECONDS)) {
94 EXECUTOR.shutdownNow();
95 }
96
97 } catch (InterruptedException e) {
98 EXECUTOR.shutdownNow();
99 Thread.currentThread().interrupt();
100 }
101 }
102 }
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117 public ResponseContext cancelDocumentProcessing(RequestContext request, UUID docId) {
118 if (!ControllerUtil.checkParam(docId)) {
119 return Response.error(400, "Invalid request parameter Doc Id: " + docId);
120 }
121
122 String query = Log.format("docId:\"{}\"", docId);
123
124 try {
125 QueryResponse response = AthIndex.getMany(Const.SOLR_CORE_ATH_DOCS, query, null,
126 QueryResponse.class);
127
128 if (response.getResults().isEmpty()) {
129 return Response.error(404, "Document not found, docId: " + docId);
130 }
131
132 SolrDocument doc = response.getResults().get(0);
133 String status = SolrUtil.safeGetField(doc, Const.ATH_PROP_STATUS, null);
134
135
136 if (DocumentStatus.EXPORT_COMPLETED.toString().equals(status)) {
137 ConflictResponse conflict = new ConflictResponse();
138 conflict.setError("Cannot cancel processing");
139 conflict.setStatus(DocumentStatus.EXPORT_COMPLETED);
140 conflict.setDocId(docId);
141 conflict.setMessage("Document processing has already completed successfully");
142
143 return Response.builder()
144 .status(Status.CONFLICT)
145 .entity(conflict)
146 .build();
147 }
148
149 if (DocumentStatus.FAILED.toString().equals(status)) {
150 ConflictResponse conflict = new ConflictResponse();
151 conflict.setError("Cannot cancel processing");
152 conflict.setStatus(DocumentStatus.FAILED);
153 conflict.setDocId(docId);
154 conflict.setMessage("Document processing has already failed");
155
156 return Response.builder()
157 .status(Status.CONFLICT)
158 .entity(conflict)
159 .build();
160 }
161
162 if (DocumentStatus.CANCELLED.toString().equals(status)) {
163 ConflictResponse conflict = new ConflictResponse();
164 conflict.setError("Cannot cancel processing");
165 conflict.setStatus(DocumentStatus.CANCELLED);
166 conflict.setDocId(docId);
167 conflict.setMessage("Document processing has already been cancelled");
168
169 return Response.builder()
170 .status(Status.CONFLICT)
171 .entity(conflict)
172 .build();
173 }
174
175 if (DocumentStatus.IMPORT_COMPLETED.toString().equals(status)) {
176 ConflictResponse conflict = new ConflictResponse();
177 conflict.setError("Cannot cancel processing");
178 conflict.setStatus(DocumentStatus.IMPORT_COMPLETED);
179 conflict.setDocId(docId);
180 conflict.setMessage("No active processing operation to cancel");
181
182 return Response.builder()
183 .status(Status.CONFLICT)
184 .entity(conflict)
185 .build();
186 }
187
188
189 SolrInputDocument updateDoc = SolrUtil.toInputDocument(doc);
190 updateDoc.setField(Const.ATH_PROP_STATUS, DocumentStatus.CANCELLED.toString());
191 updateDoc.setField(Const.ATH_PROP_FINISHED_AT, new Date());
192 updateDoc.setField(Const.ATH_PROP_UPDATED_AT, new Date());
193
194 AthIndex.getSolr().getClient().add(Const.SOLR_CORE_ATH_DOCS, updateDoc);
195 AthIndex.getSolr().getClient().commit(Const.SOLR_CORE_ATH_DOCS);
196
197 ControllerUtil.stopPipeline(docId);
198
199 return Response.success(204, "Document processing cancelled successfully");
200
201 } catch (Exception e) {
202 return Response.error(500, "Error cancelling document processing -- " + e.getMessage());
203 }
204 }
205
206 public ResponseContext getDocuments(RequestContext request, Integer page, Integer pageSize) {
207 try {
208 SolrClient solrClient = AthIndex.getSolr().getClient();
209
210 SolrQuery q = new SolrQuery("*:*")
211 .setFields(Const.ATH_PROP_DOC_ID)
212 .addSort("_docid_", SolrQuery.ORDER.asc)
213 .setRows(Integer.MAX_VALUE);
214
215 QueryResponse response = solrClient.query(Const.SOLR_CORE_ATH_DOCS, q);
216
217 List<String> allDocIds = response.getResults().stream()
218 .map(doc -> doc.getFieldValue(Const.ATH_PROP_DOC_ID))
219 .filter(Objects::nonNull)
220 .map(Object::toString)
221 .collect(Collectors.toList());
222
223 DocumentInfosWrapper wrapper = new DocumentInfosWrapper();
224
225
226 if (allDocIds.isEmpty()) {
227 wrapper.documents(new ArrayList<>())
228 .pagination(new PaginationInfo()
229 .page(1)
230 .pageSize(0)
231 .totalItems(0L)
232 .totalPages(0)
233 .hasNext(false)
234 .hasPrevious(false));
235
236 return Response.success(200, wrapper);
237 }
238
239 long totalItems = allDocIds.size();
240 int size = (pageSize != null) ? Math.max(1, Math.min(100, pageSize)) : (int) totalItems;
241 int totalPages = (int) Math.ceil(totalItems / (double) size);
242
243 int pageNum;
244 List<String> docIdsToProcess;
245
246 if (page == null && pageSize == null) {
247
248 pageNum = 1;
249 int end = Math.min(size, allDocIds.size());
250 docIdsToProcess = allDocIds.subList(0, end);
251 totalPages = (int) Math.ceil(totalItems / (double) size);
252
253 } else {
254 pageNum = (page != null) ? Math.max(1, Math.min(page, Math.max(1, totalPages))) : 1;
255 int start = (pageNum - 1) * size;
256 int end = Math.min(start + size, allDocIds.size());
257 docIdsToProcess = allDocIds.subList(start, end);
258 }
259
260 List<DocumentInfo> resultDocs = docIdsToProcess.stream()
261 .map(this::getDocumentInfoInternal)
262 .filter(Objects::nonNull)
263 .collect(Collectors.toList());
264
265 PaginationInfo pagination = new PaginationInfo()
266 .page(pageNum)
267 .pageSize(size)
268 .totalItems(totalItems)
269 .totalPages(totalPages)
270 .hasNext(pageNum < totalPages)
271 .hasPrevious(pageNum > 1);
272
273 wrapper.documents(resultDocs)
274 .pagination(pagination);
275
276 return Response.success(200, wrapper);
277
278 } catch (Exception e) {
279 return Response.error(500, e, "Error fetching documents");
280 }
281 }
282
283 private DocumentInfo getDocumentInfoInternal(String docId) {
284 try {
285 String docQuery = Log.format("docId:\"{}\"", docId);
286
287 SolrClient solrClient = AthIndex.getSolr().getClient();
288
289
290 SolrQuery q = new SolrQuery(docQuery);
291 q.setRows(1);
292 QueryResponse docResponse = solrClient.query(Const.SOLR_CORE_ATH_DOCS, q);
293
294 if (docResponse.getResults().isEmpty()) {
295 return null;
296 }
297
298 SolrDocument doc = docResponse.getResults().get(0);
299
300
301 String segQuery = Log.format("docId:\"{}\"", docId);
302 long segmentCount = SolrUtil.getNumDocuments(Const.SOLR_CORE_ATH_DOC_SEGMENTS, segQuery);
303
304 DocumentInfo docInfo = new DocumentInfo();
305 docInfo.setDocId(UUID.fromString(docId));
306
307 docInfo.setDocFileName(SolrUtil.safeGetField(doc, Const.ATH_PROP_DOC_FILE_NAME, null));
308
309 docInfo.setDocGcsUrl(
310 AthUtil.toURI(SolrUtil.safeGetField(doc, Const.ATH_PROP_DOC_STORAGE_NAME, "")));
311
312 docInfo.setDocTrlGcsUrl(
313 AthUtil.toURI(SolrUtil.safeGetField(doc, Const.ATH_PROP_DOC_TRL_STORAGE_NAME, "")));
314
315 docInfo.setSrcLang(SolrUtil.safeGetField(doc, Const.ATH_PROP_SRC_LANG, "en"));
316 docInfo.setTrgLang(SolrUtil.safeGetField(doc, Const.ATH_PROP_TRG_LANG, "fr"));
317
318 docInfo.setSegmentsCount(segmentCount);
319
320 docInfo.setCreatedBy(AthUtil.safeToUuid(
321 SolrUtil.safeGetField(doc, Const.ATH_PROP_CREATED_BY, null), null));
322
323 docInfo.setUpdatedBy(AthUtil.safeToUuid(
324 SolrUtil.safeGetField(doc, Const.ATH_PROP_UPDATED_BY, null), null));
325
326 docInfo.setCreatedAt(AthUtil.safeToDate(doc.get(Const.ATH_PROP_CREATED_AT), null));
327 docInfo.setUpdatedAt(AthUtil.safeToDate(doc.get(Const.ATH_PROP_UPDATED_AT), null));
328
329 return docInfo;
330
331 } catch (Exception e) {
332 Log.error(this.getClass(), e, "Error getting document info for docId: {}", docId);
333 return null;
334 }
335 }
336
337 public ResponseContext importDocument(RequestContext request, UUID docId, JsonNode bodyNode) {
338
339 if (!ControllerUtil.checkParam(docId)) {
340 return Response.error(400, "Invalid request parameter Doc Id: " + docId);
341 }
342
343 if (!ControllerUtil.checkParam(bodyNode)) {
344 return Response.error(400, "Invalid request parameter, bodyNode is null");
345 }
346
347 ImportDocumentRequest body = AthUtil.safeFromJsonNode(bodyNode,
348 ImportDocumentRequest.class, null);
349
350 if (body == null) {
351 return Response.error(400, "Invalid request body");
352 }
353
354 String srcLang = body.getSrcLang();
355 String trgLang = body.getTrgLang();
356 String filterId = body.getFilterId();
357 String filterParams = body.getFilterParams();
358 String srcSrx = body.getSrcSrx();
359 UUID tmId = body.getTmId();
360 Integer tmThreshold = body.getTmThreshold();
361 String mtEngineId = body.getMtEngineId();
362 String mtEngineParams = body.getMtEngineParams();
363
364 List<MtResources> mtCustomResources = body.getMtCustomResources();
365
366 Boolean mtProvideConfidenceScores = body.getMtProvideConfidenceScores();
367 Boolean mtUseTranslateLlm = body.getMtUseTranslateLlm();
368 Boolean mtSendPlainText = body.getMtSendPlainText();
369 Boolean useCodesReinsertionModel = body.getUseCodesReinsertionModel();
370 String codesReinsertionModelName = body.getCodesReinsertionModelName();
371 UUID userId = body.getUserId();
372
373 if (!ControllerUtil.checkParam(srcLang)) {
374 return Response.error(400, "Invalid request, srcLang is not specified");
375 }
376
377 if (!ControllerUtil.checkParam(trgLang)) {
378 return Response.error(400, "Invalid request, trgLang is not specified");
379 }
380
381 if (!ControllerUtil.checkParam(userId)) {
382 return Response.error(400, "Invalid request parameter User Id: " + userId);
383 }
384
385 String query = Log.format("docId:\"{}\"", docId);
386
387 try {
388 QueryResponse response = AthIndex.getMany(Const.SOLR_CORE_ATH_DOCS, query, null,
389 QueryResponse.class);
390
391 if (response.getResults().isEmpty()) {
392 return Response.error(404, "Document not found, docId: " + docId);
393 }
394
395 SolrDocument existingDoc = response.getResults().get(0);
396 String status = SolrUtil.safeGetField(existingDoc, Const.ATH_PROP_STATUS, null);
397
398 if (DocumentStatus.IMPORTING.toString().equals(status)
399 || DocumentStatus.EXPORTING.toString().equals(status)) {
400
401 ConflictResponse conflict = new ConflictResponse();
402 conflict.setError("Processing already in progress");
403 conflict.setStatus(AthUtil.safeToEnum(status, DocumentStatus.class, null));
404 conflict.setDocId(docId);
405 conflict.setStatusUrl(Log.format("/document/{}/status", docId));
406
407 return Response.builder()
408 .status(Status.CONFLICT)
409 .header("Location", Log.format("/document/{}/status", docId))
410 .entity(conflict)
411 .build();
412 }
413
414 DocumentInfo docInfo = getDocumentInfoInternal(docId.toString());
415
416
417 boolean newDoc = docInfo.getSegmentsCount() == 0;
418
419
420 String docFileName = SolrUtil.safeGetField(existingDoc, Const.ATH_PROP_DOC_FILE_NAME, null);
421 String docGcsUrl = SolrUtil.safeGetField(existingDoc, Const.ATH_PROP_DOC_STORAGE_NAME, null);
422
423 String docEncoding = SolrUtil.safeGetField(existingDoc, Const.ATH_PROP_DOC_FILE_ENCODING,
424 null);
425
426
427 SolrInputDocument doc = SolrUtil.toInputDocument(existingDoc);
428
429 doc.setField(Const.ATH_PROP_SRC_LANG, srcLang);
430 doc.setField(Const.ATH_PROP_TRG_LANG, trgLang);
431 doc.setField(Const.ATH_PROP_FILTER_ID, filterId);
432
433 SolrUtil.safeSetField(doc, Const.ATH_PROP_FILTER_PARAMS, filterParams);
434 SolrUtil.safeSetField(doc, Const.ATH_PROP_SRC_SRX, srcSrx);
435 SolrUtil.safeSetField(doc, Const.ATH_PROP_TM_ID, tmId);
436 doc.setField(Const.ATH_PROP_TM_THRESHOLD, tmThreshold != null ? tmThreshold : 75);
437
438 SolrUtil.safeSetField(doc, Const.ATH_PROP_MT_ENGINE_ID, mtEngineId);
439 SolrUtil.safeSetField(doc, Const.ATH_PROP_MT_ENGINE_PARAMS, mtEngineParams);
440
441
442
443
444
445
446
447
448
449
450 doc.setField(Const.ATH_PROP_STATUS, DocumentStatus.IMPORTING.toString());
451 doc.setField(Const.ATH_PROP_PROCESSED_BY, userId.toString());
452 doc.setField(Const.ATH_PROP_STARTED_AT, new Date());
453 doc.setField(Const.ATH_PROP_FINISHED_AT, null);
454 doc.setField(Const.ATH_PROP_UPDATED_BY, userId.toString());
455 doc.setField(Const.ATH_PROP_UPDATED_AT, new Date());
456
457 AthIndex.getSolr().getClient().add(Const.SOLR_CORE_ATH_DOCS, doc);
458 AthIndex.getSolr().getClient().commit(Const.SOLR_CORE_ATH_DOCS);
459
460
461 EXECUTOR.submit(() -> {
462 ResponseContext res = Response.success(200);
463
464 try {
465
466 if (ControllerUtil.isCancelled(docId)) {
467 Log.info(getClass(), "Import cancelled before starting for docId: {}", docId);
468 return false;
469 }
470
471 res = ControllerUtil.importFile(
472 doc,
473 docId,
474 docFileName,
475 AthUtil.toURI(docGcsUrl),
476 docEncoding,
477 srcLang,
478 trgLang,
479 filterId,
480 filterParams,
481 srcSrx,
482 tmId,
483 tmThreshold,
484 mtEngineId,
485 mtEngineParams,
486 mtCustomResources,
487 mtProvideConfidenceScores,
488 mtUseTranslateLlm != null ? mtUseTranslateLlm : false,
489 mtSendPlainText != null ? mtSendPlainText : false,
490 useCodesReinsertionModel != null ? useCodesReinsertionModel : false,
491 codesReinsertionModelName,
492 userId,
493 newDoc);
494
495
496 if (ControllerUtil.isCancelled(docId) || res.getStatus() == 499) {
497 doc.setField(Const.ATH_PROP_STATUS, DocumentStatus.CANCELLED.toString());
498 Log.info(getClass(), "Import cancelled during execution for docId: {}", docId);
499
500 } else if (res.getStatus() == 200) {
501 doc.setField(Const.ATH_PROP_STATUS, DocumentStatus.IMPORT_COMPLETED.toString());
502
503 } else {
504 doc.setField(Const.ATH_PROP_STATUS, DocumentStatus.FAILED.toString());
505 doc.setField(Const.ATH_PROP_ERROR_MESSAGE, Response.getMessage(res));
506 }
507
508 } catch (Exception e) {
509 Log.error(getClass(), e, "Import failed");
510
511
512 if (ControllerUtil.isCancelled(docId) || res.getStatus() == 499) {
513 doc.setField(Const.ATH_PROP_STATUS, DocumentStatus.CANCELLED.toString());
514
515 } else {
516 doc.setField(Const.ATH_PROP_STATUS, DocumentStatus.FAILED.toString());
517 doc.setField(Const.ATH_PROP_ERROR_MESSAGE, e.getMessage());
518 }
519
520 res = Response.error(500, e, "Import error");
521
522 } finally {
523 doc.setField(Const.ATH_PROP_FINISHED_AT, new Date());
524 doc.setField(Const.ATH_PROP_UPDATED_BY, userId.toString());
525 doc.setField(Const.ATH_PROP_UPDATED_AT, new Date());
526
527 try {
528 AthIndex.getSolr().getClient().add(Const.SOLR_CORE_ATH_DOCS, doc);
529 AthIndex.getSolr().getClient().commit(Const.SOLR_CORE_ATH_DOCS);
530
531 } catch (Exception e) {
532 Log.error(getClass(), e, "Failed to update Solr after import");
533 }
534 }
535
536 return res.getStatus() == 200 || res.getStatus() == 499;
537 });
538
539 ProcessingResponse processingResponse = new ProcessingResponse();
540 processingResponse.setStatus(ProcessingResponse.StatusEnum.IMPORTING);
541 processingResponse.setDocId(docId);
542 processingResponse.setStatusUrl(Log.format("/document/{}/status", docId));
543 processingResponse.setSubmittedAt(new Date());
544
545 return Response.builder()
546 .status(Status.ACCEPTED)
547 .header("Location", Log.format("/document/{}/status", docId))
548 .entity(processingResponse)
549 .build();
550
551 } catch (Exception e) {
552 return Response.error(500, "Error importing document -- " + e.getMessage());
553 }
554 }
555
556 public ResponseContext exportDocument(RequestContext request, UUID docId, JsonNode bodyNode) {
557
558 ExportDocumentRequest body = AthUtil.safeFromJsonNode(bodyNode, ExportDocumentRequest.class,
559 null);
560
561 if (body == null) {
562 return Response.error(400, "Invalid request body");
563 }
564
565 URI docOutGcsUrl = body.getDocOutGcsUrl();
566 String docOutEncoding = body.getDocOutEncoding();
567 UUID tmId = body.getTmId();
568 Boolean adjustTargetPages = body.getAdjustTargetPages();
569 UUID userId = body.getUserId();
570
571 if (!ControllerUtil.checkParam(docId)) {
572 return Response.error(400, "Invalid request parameter Doc Id: " + docId);
573 }
574
575 if (!ControllerUtil.checkParam(docOutGcsUrl)) {
576 return Response.error(400, "Invalid request, docOutStorageName is not specified");
577 }
578
579 if (!ControllerUtil.checkParam(userId)) {
580 return Response.error(400, "Invalid request parameter User Id: " + userId);
581 }
582
583 String query = Log.format("docId:\"{}\"", docId);
584
585 try {
586 QueryResponse response = AthIndex.getMany(Const.SOLR_CORE_ATH_DOCS, query, null,
587 QueryResponse.class);
588
589 if (response.getResults().isEmpty()) {
590 return Response.error(404, "Document not found, docId: " + docId);
591 }
592
593 SolrDocument doc = response.getResults().get(0);
594
595 String status = SolrUtil.safeGetField(doc, Const.ATH_PROP_STATUS, null);
596
597
598 if (DocumentStatus.IMPORTING.toString().equals(status)) {
599 ConflictResponse conflict = new ConflictResponse();
600 conflict.setError("Cannot start export");
601 conflict.setStatus(DocumentStatus.IMPORTING);
602 conflict.setDocId(docId);
603 conflict.setMessage("Import must be completed before export can be triggered");
604 conflict.setStatusUrl(Log.format("/document/{}/status", docId));
605
606 return Response.builder()
607 .status(Status.CONFLICT)
608 .entity(conflict)
609 .build();
610 }
611
612 if (DocumentStatus.EXPORTING.toString().equals(status)) {
613 ConflictResponse conflict = new ConflictResponse();
614 conflict.setError("Export already in progress");
615 conflict.setStatus(DocumentStatus.EXPORTING);
616 conflict.setDocId(docId);
617 conflict.setStatusUrl(Log.format("/document/{}/status", docId));
618
619 return Response.builder()
620 .status(Status.CONFLICT)
621 .entity(conflict)
622 .build();
623 }
624
625 SolrDocument tmDoc = null;
626
627 if (tmId != null) {
628 tmDoc = SolrUtil.getTmByTmId(tmId);
629
630 if (tmDoc == null) {
631 return Response.error(404,
632 "TM not found, we can import to an existing TM only (tmId = {})", tmId);
633 }
634 }
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649 SolrInputDocument updateDoc = SolrUtil.toInputDocument(doc);
650
651 String docFileName = doc.getFieldValue(Const.ATH_PROP_DOC_FILE_NAME).toString();
652
653 updateDoc.setField(Const.ATH_PROP_STATUS, DocumentStatus.EXPORTING.toString());
654 updateDoc.setField(Const.ATH_PROP_PROCESSED_BY, userId.toString());
655 updateDoc.setField(Const.ATH_PROP_STARTED_AT, new Date());
656 updateDoc.setField(Const.ATH_PROP_FINISHED_AT, null);
657 updateDoc.setField(Const.ATH_PROP_ERROR_MESSAGE, null);
658 updateDoc.setField(Const.ATH_PROP_ERROR_TYPE, null);
659
660 updateDoc.setField(Const.ATH_PROP_DOC_OUT_STORAGE_NAME, docOutGcsUrl.toString());
661
662 if (docOutEncoding == null) {
663 docOutEncoding = doc.getFieldValue(Const.ATH_PROP_DOC_OUT_FILE_ENCODING) != null
664 ? doc.getFieldValue(Const.ATH_PROP_DOC_OUT_FILE_ENCODING).toString()
665 : null;
666
667 if (docOutEncoding == null) {
668
669 docOutEncoding = doc.getFieldValue(Const.ATH_PROP_DOC_FILE_ENCODING) != null
670 ? doc.getFieldValue(Const.ATH_PROP_DOC_FILE_ENCODING).toString()
671 : null;
672 }
673 }
674
675 updateDoc.setField(Const.ATH_PROP_DOC_OUT_FILE_ENCODING, docOutEncoding);
676
677 if (tmId != null) {
678 updateDoc.setField(Const.ATH_PROP_EXPORT_TM_ID, tmId.toString());
679 }
680
681 AthIndex.getSolr().getClient().add(Const.SOLR_CORE_ATH_DOCS, updateDoc);
682 AthIndex.getSolr().getClient().commit(Const.SOLR_CORE_ATH_DOCS);
683
684
685
686
687
688
689
690
691
692 final SolrDocument tmDocRef = tmDoc;
693 final String outEnc = docOutEncoding;
694
695 EXECUTOR.submit(() -> {
696 ResponseContext res = Response.success(200);
697
698 try {
699
700 if (ControllerUtil.isCancelled(docId)) {
701 Log.info(getClass(), "Export cancelled before starting for docId: {}", docId);
702
703 updateDoc.setField(Const.ATH_PROP_STATUS, DocumentStatus.CANCELLED.toString());
704 updateDoc.setField(Const.ATH_PROP_FINISHED_AT, new Date());
705
706 AthIndex.getSolr().getClient().add(Const.SOLR_CORE_ATH_DOCS, updateDoc);
707 AthIndex.getSolr().getClient().commit(Const.SOLR_CORE_ATH_DOCS);
708 return false;
709 }
710
711 res = ControllerUtil.exportFile(doc, tmDocRef, updateDoc, docOutGcsUrl, outEnc,
712 tmId, adjustTargetPages != null ? adjustTargetPages : false, userId);
713
714
715
716 if (ControllerUtil.isCancelled(docId) || res.getStatus() == 499) {
717 updateDoc.setField(Const.ATH_PROP_STATUS, DocumentStatus.CANCELLED.toString());
718 Log.info(getClass(), "Export cancelled during execution for docId: {}", docId);
719
720 } else if (res.getStatus() == 200) {
721 Log.info(getClass(), "Export of '{}' succeeded", docFileName);
722 updateDoc.setField(Const.ATH_PROP_STATUS, DocumentStatus.EXPORT_COMPLETED.toString());
723
724 } else {
725 Log.warn(getClass(), Response.getMessage(res));
726 updateDoc.setField(Const.ATH_PROP_STATUS, DocumentStatus.FAILED.toString());
727 updateDoc.setField(Const.ATH_PROP_ERROR_MESSAGE, Response.getMessage(res));
728 }
729
730 } catch (Exception e) {
731 Log.error(DocumentController.class, e, "Export error");
732
733 if (ControllerUtil.isCancelled(docId) || res.getStatus() == 499) {
734 updateDoc.setField(Const.ATH_PROP_STATUS, DocumentStatus.CANCELLED.toString());
735
736 } else {
737 updateDoc.setField(Const.ATH_PROP_STATUS, DocumentStatus.FAILED.toString());
738 updateDoc.setField(Const.ATH_PROP_ERROR_MESSAGE, e.getMessage());
739 }
740
741 res = Response.error(500, e, "Export error");
742 }
743
744 updateDoc.setField(Const.ATH_PROP_FINISHED_AT, new Date());
745
746 AthIndex.getSolr().getClient().add(Const.SOLR_CORE_ATH_DOCS, updateDoc);
747 AthIndex.getSolr().getClient().commit(Const.SOLR_CORE_ATH_DOCS);
748
749 return res.getStatus() == 200 || res.getStatus() == 499;
750 });
751
752 ProcessingResponse processingResponse = new ProcessingResponse();
753 processingResponse.setStatus(ProcessingResponse.StatusEnum.EXPORTING);
754 processingResponse.setDocId(docId);
755 processingResponse.setStatusUrl(Log.format("/document/{}/status", docId));
756 processingResponse.setSubmittedAt(new Date());
757
758 return Response.builder()
759 .status(Status.ACCEPTED)
760 .header("Location", Log.format("/document/{}/status", docId))
761 .entity(processingResponse)
762 .build();
763
764 } catch (Exception e) {
765 return Response.error(500, "Error starting document export -- " + e.getMessage());
766 }
767 }
768
769
770
771 public ResponseContext getDocumentStatus(RequestContext request, UUID docId) {
772 if (!ControllerUtil.checkParam(docId)) {
773 return Response.error(400, "Invalid request parameter Doc Id: " + docId);
774 }
775
776 String query = Log.format("docId:\"{}\"", docId);
777
778 try {
779 QueryResponse response = AthIndex.getMany(Const.SOLR_CORE_ATH_DOCS, query, null,
780 QueryResponse.class);
781
782 if (response.getResults().isEmpty()) {
783 return Response.error(404, "Document not found, docId: " + docId);
784 }
785
786 SolrDocument doc = response.getResults().get(0);
787 String status = SolrUtil.safeGetField(doc, Const.ATH_PROP_STATUS, null);
788
789 if (DocumentStatus.IMPORTING.toString().equals(status)
790 || DocumentStatus.EXPORTING.toString().equals(status)) {
791
792 ProcessingStatusResponse processingResponse = new ProcessingStatusResponse();
793 processingResponse.setStatus(AthUtil.safeToEnum(status,
794 ProcessingStatusResponse.StatusEnum.class, null));
795 processingResponse.setDocId(docId);
796
797 Date startedAt = AthUtil.safeToDate(doc.get(Const.ATH_PROP_STARTED_AT), null);
798 processingResponse.setStartedAt(startedAt);
799
800 return Response.builder()
801 .status(Status.ACCEPTED)
802 .header("Retry-After", "30")
803 .entity(processingResponse)
804 .build();
805
806 } else if (DocumentStatus.IMPORT_COMPLETED.toString().equals(status)) {
807
808 ImportCompletedStatusResponse completedResponse = new ImportCompletedStatusResponse();
809 completedResponse.setStatus(AthUtil.safeToEnum(status,
810 ImportCompletedStatusResponse.StatusEnum.class,
811 ImportCompletedStatusResponse.StatusEnum.IMPORT_COMPLETED));
812 completedResponse.setDocId(docId);
813
814 Date finishedAt = AthUtil.safeToDate(doc.get(Const.ATH_PROP_FINISHED_AT), null);
815 completedResponse.setCompletedAt(finishedAt);
816 completedResponse.setExportUrl(Log.format("/document/{}/export", docId));
817
818 return Response.success(200, completedResponse);
819
820 } else if (DocumentStatus.EXPORT_COMPLETED.toString().equals(status)) {
821
822 ExportCompletedStatusResponse exportResponse = new ExportCompletedStatusResponse();
823 exportResponse.setStatus(AthUtil.safeToEnum(status,
824 ExportCompletedStatusResponse.StatusEnum.class,
825 ExportCompletedStatusResponse.StatusEnum.EXPORT_COMPLETED));
826 exportResponse.setDocId(docId);
827
828 Date finishedAt = AthUtil.safeToDate(doc.get(Const.ATH_PROP_FINISHED_AT), null);
829 exportResponse.setCompletedAt(finishedAt);
830 exportResponse.setDownloadUrl(Log.format("/document/{}", docId));
831
832 return Response.success(200, exportResponse);
833
834 } else if (DocumentStatus.FAILED.toString().equals(status)) {
835
836 FailedStatusResponse failedResponse = new FailedStatusResponse();
837 failedResponse.setStatus(FailedStatusResponse.StatusEnum.FAILED);
838 failedResponse.setDocId(docId);
839
840 String errorMessage = doc.getFieldValue(Const.ATH_PROP_ERROR_MESSAGE) != null
841 ? doc.getFieldValue(Const.ATH_PROP_ERROR_MESSAGE).toString()
842 : "Unknown error";
843 failedResponse.setErrorMessage(errorMessage);
844
845 String errorType = SolrUtil.safeGetField(doc, Const.ATH_PROP_ERROR_TYPE,
846 "UNKNOWN_ERROR");
847
848 failedResponse.setErrorType(AthUtil.safeToEnum(errorType,
849 FailedStatusResponse.ErrorTypeEnum.class,
850 FailedStatusResponse.ErrorTypeEnum.UNKNOWN_ERROR));
851
852 Date finishedAt = AthUtil.safeToDate(doc.get(Const.ATH_PROP_FINISHED_AT), null);
853 failedResponse.setFailedAt(finishedAt);
854
855 return Response.success(200, failedResponse);
856
857 } else if (DocumentStatus.CANCELLED.toString().equals(status)) {
858
859 CancelledStatusResponse cancelledResponse = new CancelledStatusResponse();
860 cancelledResponse.setStatus(CancelledStatusResponse.StatusEnum.CANCELLED);
861 cancelledResponse.setDocId(docId);
862
863 Date finishedAt = AthUtil.safeToDate(doc.get(Const.ATH_PROP_FINISHED_AT), null);
864 cancelledResponse.setCancelledAt(finishedAt);
865
866
867 String cancelledDuring = inferCancelledPhase(docId);
868 if (cancelledDuring != null) {
869 try {
870 cancelledResponse.setCancelledDuring(
871 CancelledStatusResponse.CancelledDuringEnum.valueOf(cancelledDuring));
872
873 } catch (IllegalArgumentException e) {
874 Log.warn(getClass(), "Invalid inferred phase: {}", cancelledDuring);
875 }
876 }
877
878 return Response.success(200, cancelledResponse);
879
880 } else {
881
882 return Response.error(500, "Unknown document status: " + status);
883 }
884
885 } catch (Exception e) {
886 return Response.error(500, "Error getting document status -- " + e.getMessage());
887 }
888 }
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904 private String inferCancelledPhase(UUID docId) {
905 try {
906
907 String query = Log.format("docId:\"{}\"", docId);
908 long segmentCount = SolrUtil.getNumDocuments(Const.SOLR_CORE_ATH_DOC_SEGMENTS, query);
909
910 if (segmentCount > 0) {
911
912 return DocumentStatus.EXPORTING.toString();
913
914 } else {
915
916 return DocumentStatus.IMPORTING.toString();
917 }
918
919 } catch (Exception e) {
920 Log.error(getClass(), e, "Error inferring cancelled phase for docId: {}", docId);
921 return null;
922 }
923 }
924
925
926
927 public ResponseContext downloadDocument(RequestContext request, UUID docId) {
928 if (!ControllerUtil.checkParam(docId)) {
929 return Response.error(400, "Invalid request parameter Doc Id: " + docId);
930 }
931
932 String query = Log.format("docId:\"{}\"", docId);
933
934 try {
935 QueryResponse response = AthIndex.getMany(Const.SOLR_CORE_ATH_DOCS, query, null,
936 QueryResponse.class);
937
938 if (response.getResults().isEmpty()) {
939 return Response.error(404, "Document not found, docId: " + docId);
940 }
941
942 SolrDocument doc = response.getResults().get(0);
943
944 String status = SolrUtil.safeGetField(doc, Const.ATH_PROP_STATUS, null);
945
946 if (DocumentStatus.EXPORT_COMPLETED.toString().equals(status)) {
947
948 String docFileName = SolrUtil.safeGetField(doc, Const.ATH_PROP_DOC_FILE_NAME,
949 "download");
950
951 String docOutStorageName = SolrUtil.safeGetField(doc,
952 Const.ATH_PROP_DOC_OUT_STORAGE_NAME, null);
953
954 if (docOutStorageName == null) {
955 return Response.error(500,
956 "Cannot create the export file -- GCS storage name (doc_out_storage_name) is not "
957 + "specified");
958 }
959
960 File exportedFile = null;
961
962 try {
963 exportedFile = AthUtil.createTempFile();
964
965
966 AthStorage.loadFile(AthUtil.toURI(docOutStorageName), exportedFile);
967
968 } catch (IOException e) {
969 return Response.error(500, "Cannot create temp file -- " + e.getMessage());
970 }
971
972 return Response.builder()
973 .status(Status.OK)
974 .header("Content-Disposition",
975 Log.format("attachment; filename=\"{}\"", docFileName))
976 .entity(exportedFile)
977 .build();
978
979 } else if (DocumentStatus.IMPORTING.toString().equals(status)) {
980 ConflictDownloadResponse conflict = new ConflictDownloadResponse();
981 conflict.setError("Document not ready for download");
982 conflict.setStatus(AthUtil.safeToEnum(status,
983 ConflictDownloadResponse.StatusEnum.class, null));
984 conflict.setMessage("Document import is still in progress. Check status endpoint.");
985 conflict.setStatusUrl(Log.format("/document/{}/status", docId));
986
987 return Response.builder()
988 .status(Status.CONFLICT)
989 .entity(conflict)
990 .build();
991
992 } else if (DocumentStatus.IMPORT_COMPLETED.toString().equals(status)) {
993 ConflictDownloadResponse conflict = new ConflictDownloadResponse();
994 conflict.setError("Document not ready for download");
995 conflict.setStatus(AthUtil.safeToEnum(status,
996 ConflictDownloadResponse.StatusEnum.class, null));
997 conflict.setMessage("Call POST /document/{doc_id}/export to generate the translation");
998 conflict.setStatusUrl(Log.format("/document/{}/status", docId));
999 conflict.setExportUrl(Log.format("/document/{}/export", docId));
1000
1001 return Response.builder()
1002 .status(Status.CONFLICT)
1003 .entity(conflict)
1004 .build();
1005
1006 } else if (DocumentStatus.EXPORTING.toString().equals(status)) {
1007 ConflictDownloadResponse conflict = new ConflictDownloadResponse();
1008 conflict.setError("Document not ready for download");
1009 conflict.setStatus(AthUtil.safeToEnum(status,
1010 ConflictDownloadResponse.StatusEnum.class, null));
1011 conflict.setMessage("Document export is still in progress. Check status endpoint.");
1012 conflict.setStatusUrl(Log.format("/document/{}/status", docId));
1013
1014 return Response.builder()
1015 .status(Status.CONFLICT)
1016 .entity(conflict)
1017 .build();
1018
1019 } else {
1020 return Response.error(500, "Unexpected document status: " + status);
1021 }
1022
1023 } catch (Exception e) {
1024 return Response.error(500, "Error downloading document -- " + e.getMessage());
1025 }
1026 }
1027
1028 public ResponseContext deleteDocument(RequestContext request, UUID docId) {
1029 if (!ControllerUtil.checkParam(docId)) {
1030 return Response.error(400, "Invalid request parameter Doc Id: " + docId);
1031 }
1032
1033 String query = Log.format("docId:\"{}\"", docId);
1034
1035 try {
1036
1037 if (SolrUtil.getNumDocuments(Const.SOLR_CORE_ATH_DOCS, query) <= 0) {
1038 return Response.error(404, "Document not found, docId: " + docId);
1039 }
1040
1041
1042 AthIndex.deleteByQuery(Const.SOLR_CORE_ATH_DOCS, query);
1043 AthIndex.deleteByQuery(Const.SOLR_CORE_ATH_DOC_SEGMENTS, query);
1044
1045
1046
1047 return Response.success(204, "Document (id={}) was deleted successfully", docId);
1048
1049 } catch (Exception e) {
1050 String st = Log.format("Error deleting Document (id={}) -- {}", docId, e.getMessage());
1051 Log.error(this.getClass(), e, st);
1052 return Response.error(500, st);
1053 }
1054 }
1055
1056
1057
1058 public ResponseContext getDocumentSegments(RequestContext request, UUID docId,
1059 Integer page, Integer pageSize) {
1060
1061 if (!ControllerUtil.checkParam(docId)) {
1062 return Response.error(400, "Invalid request parameter Doc Id: " + docId);
1063 }
1064
1065 String query = Log.format("docId:\"{}\"", docId);
1066
1067 try {
1068 long totalItems = SolrUtil.getNumDocuments(Const.SOLR_CORE_ATH_DOC_SEGMENTS, query);
1069
1070 if (totalItems <= 0) {
1071 DocumentSegmentsWrapper emptyWrapper = new DocumentSegmentsWrapper()
1072 .documentSegments(new ArrayList<>())
1073 .pagination(new PaginationInfo()
1074 .page(1)
1075 .pageSize(0)
1076 .totalItems(0L)
1077 .totalPages(0)
1078 .hasNext(false)
1079 .hasPrevious(false));
1080
1081 return Response.success(200, emptyWrapper);
1082 }
1083
1084
1085 int size = (pageSize != null) ? Math.max(1, Math.min(100, pageSize)) : (int) totalItems;
1086 int totalPages = (int) Math.ceil(totalItems / (double) size);
1087 int pageNum = (page != null) ? Math.max(1, Math.min(page, Math.max(1, totalPages))) : 1;
1088
1089 SolrClient solrClient = AthIndex.getSolr().getClient();
1090 SolrQuery solrQuery = new SolrQuery(query)
1091 .setStart((pageNum - 1) * size)
1092 .setRows(size)
1093 .addSort(Const.ATH_PROP_POSITION, SolrQuery.ORDER.asc);
1094
1095 QueryResponse response = solrClient.query(Const.SOLR_CORE_ATH_DOC_SEGMENTS, solrQuery);
1096 SolrDocumentList docList = response.getResults();
1097
1098 List<DocumentSegment> segments = docList.stream()
1099 .map(this::toDocumentSegment)
1100 .filter(Objects::nonNull)
1101 .collect(Collectors.toList());
1102
1103
1104
1105 PaginationInfo pagination = new PaginationInfo()
1106 .page(pageNum)
1107 .pageSize(size)
1108 .totalItems(totalItems)
1109 .totalPages(totalPages)
1110 .hasNext(pageNum < totalPages)
1111 .hasPrevious(pageNum > 1);
1112
1113 DocumentSegmentsWrapper wrapper = new DocumentSegmentsWrapper()
1114 .documentSegments(segments)
1115 .pagination(pagination);
1116
1117 return Response.success(200, wrapper);
1118
1119 } catch (Exception e) {
1120 return Response.error(500, e, "Error fetching document segments");
1121 }
1122 }
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135 public ResponseContext createDocumentSegment(RequestContext request, UUID docId,
1136 JsonNode bodyNode) {
1137
1138 if (!ControllerUtil.checkParam(docId)) {
1139 return Response.error(400, "Invalid request parameter Doc Id: " + docId);
1140 }
1141
1142 if (!ControllerUtil.checkParam(bodyNode)) {
1143 return Response.error(400, "Invalid request parameter, bodyNode is null");
1144 }
1145
1146 CreateDocumentSegmentRequest body = AthUtil.safeFromJsonNode(bodyNode,
1147 CreateDocumentSegmentRequest.class, null);
1148
1149 if (body == null) {
1150 return Response.error(400, "Invalid request body");
1151 }
1152
1153 try {
1154 LayeredTextX source = body.getSource();
1155 LayeredTextX target = body.getTarget();
1156 Long position = body.getPosition();
1157 Origin origin = body.getOrigin();
1158 UUID userId = body.getUserId();
1159 String tuId = body.getTuId();
1160
1161 SolrDocument solrDoc = SolrUtil.getDocumentByDocId(docId);
1162
1163 if (solrDoc == null) {
1164 return Response.error(404, "Document not found, docId: " + docId);
1165 }
1166
1167
1168
1169 SolrUtil.moveDocSegmentsBelow(docId, position);
1170
1171 DocumentSegment segment = new DocumentSegment();
1172
1173 segment.setSource(source);
1174 segment.setTarget(target);
1175 segment.setPosition(position);
1176 segment.setOrigin(origin);
1177
1178 segment.setId(SolrUtil.buildDocSegSolrId(docId, position));
1179 segment.setDocFileName(SolrUtil.safeGetField(solrDoc, Const.ATH_PROP_DOC_FILE_NAME, null));
1180
1181 segment.setDocId(docId);
1182 segment.setTuId(tuId);
1183 segment.setDocSegId(UUID.randomUUID());
1184 segment.setSrcLang(SolrUtil.safeGetField(solrDoc, Const.ATH_PROP_SRC_LANG, null));
1185 segment.setTrgLang(SolrUtil.safeGetField(solrDoc, Const.ATH_PROP_TRG_LANG, null));
1186
1187 segment.setCreatedBy(userId);
1188 segment.setCreatedAt(new Date());
1189
1190
1191 SolrInputDocument segDoc = toSolrDoc(segment);
1192 AthIndex.getSolr().getClient().add(Const.SOLR_CORE_ATH_DOC_SEGMENTS, segDoc);
1193 AthIndex.getSolr().getClient().commit(Const.SOLR_CORE_ATH_DOC_SEGMENTS);
1194
1195 return Response.success(201, "Document segment created successfully");
1196
1197 } catch (Exception e) {
1198 return Response.error(500, "Error creating document segment -- " + e.getMessage());
1199 }
1200 }
1201
1202 public ResponseContext getDocumentSegment(RequestContext request, UUID docId,
1203 UUID docSegId) {
1204
1205 if (!ControllerUtil.checkParam(docId)) {
1206 return Response.error(400, "Invalid request parameter Doc Id: " + docId);
1207 }
1208
1209 if (!ControllerUtil.checkParam(docSegId)) {
1210 return Response.error(400, "Invalid request parameter Segment Id: " + docSegId);
1211 }
1212
1213 String query = Log.format("docId:\"{}\" AND docSegId:\"{}\"", docId, docSegId);
1214
1215 try {
1216 QueryResponse response = AthIndex.getMany(Const.SOLR_CORE_ATH_DOC_SEGMENTS, query, null,
1217 QueryResponse.class);
1218
1219 SolrDocumentList docList = response.getResults();
1220
1221 if (docList.isEmpty()) {
1222 return Response.error(404, "Document segment not found");
1223 }
1224
1225 SolrDocument doc = docList.get(0);
1226 DocumentSegment segment = toDocumentSegment(doc);
1227
1228 if (segment == null) {
1229 return Response.error(500, "Error parsing segment data");
1230 }
1231
1232 DocumentSegmentWrapper wrapper = new DocumentSegmentWrapper();
1233 wrapper.setDocumentSegment(segment);
1234
1235 return Response.success(200, wrapper);
1236
1237 } catch (Exception e) {
1238 return Response.error(500, "Error fetching document segment -- " + e.getMessage());
1239 }
1240 }
1241
1242 public ResponseContext updateDocumentSegment(RequestContext request, UUID docId,
1243 UUID docSegId, JsonNode bodyNode) {
1244
1245 if (!ControllerUtil.checkParam(docId)) {
1246 return Response.error(400, "Invalid request parameter Doc Id: " + docId);
1247 }
1248
1249 if (!ControllerUtil.checkParam(docSegId)) {
1250 return Response.error(400, "Invalid request parameter Segment Id: " + docSegId);
1251 }
1252
1253 if (!ControllerUtil.checkParam(bodyNode)) {
1254 return Response.error(400, "Invalid request, bodyNode is null");
1255 }
1256
1257 UpdateDocumentSegmentRequest body = AthUtil.safeFromJsonNode(bodyNode,
1258 UpdateDocumentSegmentRequest.class, null);
1259
1260 if (body == null) {
1261 return Response.error(400, "Invalid request body");
1262 }
1263
1264 try {
1265 LayeredTextX target = body.getTarget();
1266 Origin origin = body.getOrigin();
1267 UUID userId = body.getUserId();
1268
1269 SolrDocument solrDoc = SolrUtil.getDocumentSegment(docId, docSegId);
1270
1271 if (solrDoc == null) {
1272 return Response.error(404, "Document segment not found (docId: '{}' docSegId: '{}')", docId,
1273 docSegId);
1274 }
1275
1276
1277 DocumentSegment segment = toDocumentSegment(solrDoc);
1278
1279 segment.setTarget(target);
1280 segment.setOrigin(origin);
1281
1282 segment.setUpdatedBy(userId);
1283 segment.setUpdatedAt(new Date());
1284
1285
1286 SolrInputDocument segDoc = toSolrDoc(segment);
1287 AthIndex.getSolr().getClient().add(Const.SOLR_CORE_ATH_DOC_SEGMENTS, segDoc);
1288 AthIndex.getSolr().getClient().commit(Const.SOLR_CORE_ATH_DOC_SEGMENTS);
1289
1290 return Response.success(200, "Document segment updated successfully");
1291
1292 } catch (Exception e) {
1293 return Response.error(500, "Error updating document segment -- " + e.getMessage());
1294 }
1295 }
1296
1297 public ResponseContext deleteDocumentSegment(RequestContext request, UUID docId,
1298 UUID docSegId) {
1299
1300 if (!ControllerUtil.checkParam(docId)) {
1301 return Response.error(400, "Invalid request parameter Doc Id: " + docId);
1302 }
1303
1304 if (!ControllerUtil.checkParam(docSegId)) {
1305 return Response.error(400, "Invalid request parameter Segment Id: " + docSegId);
1306 }
1307
1308 String query = Log.format("docId:\"{}\" AND docSegId:\"{}\"", docId, docSegId);
1309
1310 try {
1311 if (SolrUtil.getNumDocuments(Const.SOLR_CORE_ATH_DOC_SEGMENTS, query) <= 0) {
1312 return Response.error(404, "Document segment not found");
1313 }
1314
1315 AthIndex.deleteByQuery(Const.SOLR_CORE_ATH_DOC_SEGMENTS, query);
1316
1317 return Response.success(204, "Document segment deleted successfully");
1318
1319 } catch (Exception e) {
1320 String st = Log.format("Error deleting document segment (docId={}, segId={}) -- {}",
1321 docId, docSegId, e.getMessage());
1322 Log.error(this.getClass(), e, st);
1323 return Response.error(500, st);
1324 }
1325 }
1326
1327
1328
1329 private DocumentSegment toDocumentSegment(SolrDocument doc) {
1330 try {
1331 DocumentSegment segment = new DocumentSegment();
1332
1333
1334 segment.setId(SolrUtil.safeGetField(doc, Const.ATH_PROP_SOLR_ID, null));
1335
1336 segment.setDocId(AthUtil.safeToUuid(
1337 SolrUtil.safeGetField(doc, Const.ATH_PROP_DOC_ID, null), null));
1338
1339 segment.setDocSegId(AthUtil.safeToUuid(
1340 SolrUtil.safeGetField(doc, Const.ATH_PROP_DOC_SEG_ID, null), null));
1341
1342 segment.setTuId(SolrUtil.safeGetField(doc, Const.ATH_PROP_TU_ID, null));
1343
1344 segment.setDocFileName(SolrUtil.safeGetField(doc, Const.ATH_PROP_DOC_FILE_NAME, null));
1345 segment.setSrcLang(SolrUtil.safeGetField(doc, Const.ATH_PROP_SRC_LANG, null));
1346 segment.setTrgLang(SolrUtil.safeGetField(doc, Const.ATH_PROP_TRG_LANG, null));
1347
1348
1349 segment.setPosition(SolrUtil.safeGetLongField(doc, Const.ATH_PROP_POSITION, null));
1350
1351
1352 String sourceJson = doc.getFieldValue(Const.ATH_PROP_SOURCE_JSON) != null
1353 ? doc.getFieldValue(Const.ATH_PROP_SOURCE_JSON).toString()
1354 : null;
1355
1356 String targetJson = doc.getFieldValue(Const.ATH_PROP_TARGET_JSON) != null
1357 ? doc.getFieldValue(Const.ATH_PROP_TARGET_JSON).toString()
1358 : null;
1359
1360 if (sourceJson != null) {
1361 LayeredTextX slt = JacksonUtil.fromJson(sourceJson, LayeredTextX.class);
1362 segment.setSource(slt);
1363 }
1364
1365 if (targetJson != null) {
1366 LayeredTextX tlt = JacksonUtil.fromJson(targetJson, LayeredTextX.class);
1367 segment.setTarget(tlt);
1368 }
1369
1370
1371 segment.setOrigin(Origin.HT);
1372 segment.setTmMatchScore(0);
1373 segment.setMtConfidenceScore(0d);
1374
1375 segment.setOrigin(AthUtil.safeToEnum(
1376 SolrUtil.safeGetField(doc, Const.ATH_PROP_ORIGIN, null),
1377 Origin.class, null));
1378
1379 if (segment.getOrigin() == Origin.TM) {
1380 segment.setTmMatchScore(AthUtil.safeToInt(
1381 SolrUtil.safeGetField(doc, Const.ATH_PROP_TM_MATCH_SCORE, null), 0));
1382
1383 } else if (segment.getOrigin() == Origin.MT) {
1384 segment.setMtConfidenceScore(AthUtil.safeToDouble(
1385 doc.getFieldValue(Const.ATH_PROP_MT_CONFIDENCE_SCORE), 0d));
1386 }
1387
1388 String altTransJson = doc.getFieldValue(Const.ATH_PROP_ALT_TRANS_JSON) != null
1389 ? doc.getFieldValue(Const.ATH_PROP_ALT_TRANS_JSON).toString()
1390 : null;
1391
1392 if (altTransJson != null) {
1393 TypeReference<List<MtTargetInfo>> ref = new TypeReference<>() {
1394 };
1395
1396 List<MtTargetInfo> altTrans = JacksonUtil.fromJson(altTransJson, ref);
1397 segment.setMtTargets(altTrans);
1398 }
1399
1400 segment.setMtTargetIndex(SolrUtil.safeGetIntField(doc, Const.ATH_PROP_ALT_TRANS_INDEX, -1));
1401
1402 if (segment.getMtTargetIndex() == -1
1403 && segment.getMtTargets() != null
1404 && segment.getMtTargets().size() > 0) {
1405
1406 segment.setMtTargetIndex(0);
1407
1408 if (segment.getMtConfidenceScore() == 0) {
1409 MtTargetInfo mti = segment.getMtTargets().get(0);
1410 segment.setMtConfidenceScore(mti.getMtConfidenceScore());
1411 }
1412 }
1413
1414
1415 segment.setCreatedAt(AthUtil.safeToDate(doc.get(Const.ATH_PROP_CREATED_AT), null));
1416 segment.setUpdatedAt(AthUtil.safeToDate(doc.get(Const.ATH_PROP_UPDATED_AT), null));
1417
1418 segment.setCreatedBy(AthUtil.safeToUuid(
1419 SolrUtil.safeGetField(doc, Const.ATH_PROP_CREATED_BY, null), null));
1420
1421 segment.setUpdatedBy(AthUtil.safeToUuid(
1422 SolrUtil.safeGetField(doc, Const.ATH_PROP_UPDATED_BY, null), null));
1423
1424 return segment;
1425
1426 } catch (Exception e) {
1427 Log.error(this.getClass(), e, "Error converting Solr document to DocumentSegment");
1428 return null;
1429 }
1430 }
1431
1432 private SolrInputDocument toSolrDoc(DocumentSegment segment) throws AthException {
1433 SolrInputDocument doc = new SolrInputDocument();
1434
1435 if (segment.getId() != null) {
1436 doc.addField(Const.ATH_PROP_SOLR_ID, segment.getId().toString());
1437 }
1438
1439 if (segment.getDocSegId() != null) {
1440 doc.addField(Const.ATH_PROP_DOC_SEG_ID, segment.getDocSegId().toString());
1441 }
1442
1443 if (segment.getDocId() != null) {
1444 doc.addField(Const.ATH_PROP_DOC_ID, segment.getDocId().toString());
1445 }
1446
1447 doc.addField(Const.ATH_PROP_CAT_FRAMEWORK_NAME, Const.CAT_FRAMEWORK_NAME);
1448 doc.addField(Const.ATH_PROP_CAT_FRAMEWORK_VERSION, Const.CAT_FRAMEWORK_VERSION);
1449
1450 if (segment.getTuId() != null) {
1451 doc.addField(Const.ATH_PROP_TU_ID, segment.getTuId());
1452 }
1453
1454 if (segment.getDocFileName() != null) {
1455 doc.addField(Const.ATH_PROP_DOC_FILE_NAME, segment.getDocFileName());
1456 }
1457
1458 if (segment.getSrcLang() != null) {
1459 doc.addField(Const.ATH_PROP_SRC_LANG, segment.getSrcLang());
1460 }
1461
1462 if (segment.getTrgLang() != null) {
1463 doc.addField(Const.ATH_PROP_TRG_LANG, segment.getTrgLang());
1464 }
1465
1466 if (segment.getPosition() != null) {
1467 doc.addField(Const.ATH_PROP_POSITION, segment.getPosition().toString());
1468 }
1469
1470
1471 if (segment.getSource() != null) {
1472 String sourceJson = JacksonUtil.toJson(segment.getSource(), false);
1473 doc.addField(Const.ATH_PROP_SOURCE_JSON, sourceJson);
1474
1475
1476 if (segment.getSource().getText() != null) {
1477 doc.addField(Const.ATH_PROP_SOURCE, segment.getSource().getText());
1478 }
1479
1480
1481 if (segment.getSource().getTextWithCodes() != null) {
1482 doc.addField(Const.ATH_PROP_SOURCE_WITH_CODES,
1483 segment.getSource().getTextWithCodes());
1484 }
1485 }
1486
1487 if (segment.getTarget() != null) {
1488 String targetJson = JacksonUtil.toJson(segment.getTarget(), false);
1489 doc.addField(Const.ATH_PROP_TARGET_JSON, targetJson);
1490
1491
1492 if (segment.getTarget().getText() != null) {
1493 doc.addField(Const.ATH_PROP_TARGET, segment.getTarget().getText());
1494 }
1495
1496
1497 if (segment.getTarget().getTextWithCodes() != null) {
1498 doc.addField(Const.ATH_PROP_TARGET_WITH_CODES,
1499 segment.getTarget().getTextWithCodes());
1500 }
1501 }
1502
1503 if (segment.getOrigin() != null) {
1504 doc.addField(Const.ATH_PROP_ORIGIN, segment.getOrigin().name());
1505 }
1506
1507 List<MtTargetInfo> altTrans = segment.getMtTargets();
1508
1509 if (altTrans != null) {
1510 String altTransJson = JacksonUtil.toJson(altTrans, false);
1511 doc.addField(Const.ATH_PROP_ALT_TRANS_JSON, altTransJson);
1512 }
1513
1514 doc.addField(Const.ATH_PROP_ALT_TRANS_INDEX, -1);
1515
1516 if (segment.getTmMatchScore() != null) {
1517 doc.addField(Const.ATH_PROP_TM_MATCH_SCORE, segment.getTmMatchScore());
1518 }
1519
1520 if (segment.getMtConfidenceScore() != null) {
1521 doc.addField(Const.ATH_PROP_MT_CONFIDENCE_SCORE, segment.getMtConfidenceScore());
1522 }
1523
1524
1525 if (segment.getCreatedAt() != null) {
1526 doc.addField(Const.ATH_PROP_CREATED_AT, segment.getCreatedAt());
1527 }
1528
1529 if (segment.getUpdatedAt() != null) {
1530 doc.addField(Const.ATH_PROP_UPDATED_AT, segment.getUpdatedAt());
1531 }
1532
1533
1534 if (segment.getCreatedBy() != null) {
1535 doc.addField(Const.ATH_PROP_CREATED_BY, segment.getCreatedBy().toString());
1536 }
1537
1538 if (segment.getUpdatedBy() != null) {
1539 doc.addField(Const.ATH_PROP_UPDATED_BY, segment.getUpdatedBy().toString());
1540 }
1541
1542 return doc;
1543 }
1544
1545 public ResponseContext getModifiedSegmentsSummary(RequestContext request, UUID docId,
1546 Integer page, Integer pageSize) {
1547
1548 if (!ControllerUtil.checkParam(docId)) {
1549 return Response.error(400, "Invalid request parameter Doc Id: " + docId);
1550 }
1551
1552 try {
1553 SolrDocument docInfo = SolrUtil.getDocumentByDocId(docId);
1554
1555 if (docInfo == null) {
1556 return Response.error(404, "Document not found");
1557 }
1558
1559 Date docUpdatedAt = AthUtil.safeToDate(docInfo.get(Const.ATH_PROP_UPDATED_AT), null);
1560
1561
1562 String query;
1563 if (docUpdatedAt == null) {
1564
1565
1566 query = Log.format("docId:\"{}\" AND updatedAt:[* TO *]", docId);
1567 } else {
1568
1569
1570 query = Log.format("docId:\"{}\" AND updatedAt:{{} TO *}", docId,
1571 AthUtil.dateToString(Const.QUARTZ_DATE_FORMAT, docUpdatedAt));
1572 }
1573
1574 long totalItems = SolrUtil.getNumDocuments(Const.SOLR_CORE_ATH_DOC_SEGMENTS, query);
1575
1576 int pageNum = (page != null && page >= 1) ? page : 1;
1577 int size = (pageSize != null) ? Math.max(1, Math.min(100, pageSize)) : (int) totalItems;
1578 int totalPages = (int) Math.ceil((double) totalItems / size);
1579
1580 if (pageNum > totalPages && totalPages > 0) {
1581 pageNum = totalPages;
1582 }
1583
1584 SolrQuery solrQuery = new SolrQuery(query)
1585 .setRows(size)
1586 .setStart((pageNum - 1) * size)
1587 .addSort(Const.ATH_PROP_POSITION, SolrQuery.ORDER.asc);
1588
1589 QueryResponse resp = AthIndex.getSolr().getClient().query(Const.SOLR_CORE_ATH_DOC_SEGMENTS,
1590 solrQuery);
1591
1592 List<DocumentSegment> segments = resp.getResults().stream()
1593 .map(this::toDocumentSegment)
1594 .filter(Objects::nonNull)
1595 .collect(Collectors.toList());
1596
1597 PaginationInfo pagination = new PaginationInfo()
1598 .page(pageNum)
1599 .pageSize(size)
1600 .totalItems(totalItems)
1601 .totalPages(totalPages)
1602 .hasNext(pageNum < totalPages)
1603 .hasPrevious(pageNum > 1);
1604
1605 ModifiedSegmentsWrapper wrapper = new ModifiedSegmentsWrapper()
1606 .modSegments(segments)
1607 .pagination(pagination);
1608
1609 return Response.success(200, wrapper);
1610
1611 } catch (Exception e) {
1612 return Response.error(500, e, "Error fetching modified segments");
1613 }
1614 }
1615
1616 public ResponseContext clearModifiedSegmentsSummary(RequestContext request, UUID docId) {
1617 if (!ControllerUtil.checkParam(docId)) {
1618 return Response.error(400, "Invalid request parameter Doc Id: " + docId);
1619 }
1620
1621 try {
1622
1623 SolrDocument existingDoc = SolrUtil.getDocumentByDocId(docId);
1624
1625 if (existingDoc == null) {
1626 return Response.error(404, "Document not found, docId: " + docId);
1627 }
1628
1629
1630 SolrInputDocument doc = SolrUtil.toInputDocument(existingDoc);
1631 doc.setField(Const.ATH_PROP_UPDATED_AT, new Date());
1632
1633 AthIndex.getSolr().getClient().add(Const.SOLR_CORE_ATH_DOCS, doc);
1634 AthIndex.getSolr().getClient().commit(Const.SOLR_CORE_ATH_DOCS);
1635
1636 return Response.success(200, "Modified segments summary cleared successfully");
1637
1638 } catch (Exception e) {
1639 return Response.error(500, "Error clearing modified segments summary -- " + e.getMessage());
1640 }
1641 }
1642
1643 public ResponseContext createDocument(RequestContext request, File docFile, String docUrl,
1644 String docFileName, String docEncoding, UUID docId, String docGcsUrl, UUID userId) {
1645
1646
1647
1648
1649 docFileName = docFile != null
1650 ? Util.getFilename(docFile.getAbsolutePath(), true)
1651 : Util.getFilename(docGcsUrl, true);
1652
1653 if (!ControllerUtil.checkParam(docFileName)) {
1654 return Response.error(400, "Invalid request, doc_file_name is not specified");
1655 }
1656
1657 if (!ControllerUtil.checkParam(docId)) {
1658 return Response.error(400, "Invalid request parameter Doc Id: " + docId);
1659 }
1660
1661 if (!ControllerUtil.checkParam(docGcsUrl)) {
1662 return Response.error(400, "Invalid request, doc_gcs_url is not specified");
1663 }
1664
1665 if (!ControllerUtil.checkParam(userId)) {
1666 return Response.error(400, "Invalid request parameter User Id: " + userId);
1667 }
1668
1669
1670 fileToImport = null;
1671
1672 try {
1673
1674 if (docFile != null && docFile.exists()) {
1675 fileToImport = docFile;
1676 isTempFile = false;
1677 Log.info(getClass(), "Using uploaded file: {}", docFile.getAbsolutePath());
1678 }
1679
1680
1681 else if (!Util.isEmpty(docUrl)) {
1682 URI uri = AthUtil.toURI(docUrl);
1683
1684 if (!uri.getScheme().matches("^(http|https)$")) {
1685 return Response.error(400, "doc_url must be HTTP or HTTPS");
1686 }
1687
1688 File tempFile = AthUtil.createTempFile();
1689
1690 if (tempFile == null) {
1691 return Response.error(500, "Failed to create temp file for URL download");
1692 }
1693
1694 try (CloseableHttpClient client = HttpClients.createDefault()) {
1695 HttpGet httpGet = new HttpGet(docUrl);
1696
1697 client.execute(httpGet, response -> {
1698 if (response.getStatusLine().getStatusCode() >= 400) {
1699 throw new IOException("HTTP " + response.getStatusLine().getStatusCode());
1700 }
1701
1702 try (InputStream in = response.getEntity().getContent()) {
1703 FileUtils.copyInputStreamToFile(in, tempFile);
1704 }
1705
1706 return null;
1707 });
1708 }
1709
1710 fileToImport = tempFile;
1711 isTempFile = true;
1712 Log.info(getClass(), "Downloaded from URL to temp file: {}", tempFile.getAbsolutePath());
1713 }
1714
1715
1716 String query = Log.format("docId:\"{}\"", docId);
1717 SolrDocument existingDoc = null;
1718
1719 try {
1720 QueryResponse response = AthIndex.getMany(Const.SOLR_CORE_ATH_DOCS, query, null,
1721 QueryResponse.class);
1722
1723 if (!response.getResults().isEmpty()) {
1724 existingDoc = response.getResults().get(0);
1725 }
1726
1727 } catch (Exception e) {
1728 Log.error(this.getClass(), e, "Error checking document existence");
1729 }
1730
1731
1732 if (fileToImport != null) {
1733 AthStorage.storeFile(AthUtil.toURI(docGcsUrl), OkapiUtil.getMimeType(docGcsUrl),
1734 fileToImport);
1735 }
1736
1737
1738 SolrInputDocument doc = existingDoc == null ? new SolrInputDocument()
1739 : SolrUtil.toInputDocument(existingDoc);
1740
1741 doc.setField(Const.ATH_PROP_DOC_ID, docId.toString());
1742 doc.setField(Const.ATH_PROP_CAT_FRAMEWORK_NAME, Const.CAT_FRAMEWORK_NAME);
1743 doc.setField(Const.ATH_PROP_CAT_FRAMEWORK_VERSION, Const.CAT_FRAMEWORK_VERSION);
1744 doc.setField(Const.ATH_PROP_DOC_FILE_NAME, docFileName);
1745 doc.setField(Const.ATH_PROP_DOC_STORAGE_NAME, docGcsUrl.toString());
1746 doc.setField(Const.ATH_PROP_DOC_FILE_ENCODING, docEncoding);
1747 doc.setField(Const.ATH_PROP_CREATED_BY, userId.toString());
1748 doc.setField(Const.ATH_PROP_CREATED_AT, new Date());
1749
1750 AthIndex.getSolr().getClient().add(Const.SOLR_CORE_ATH_DOCS, doc);
1751 AthIndex.getSolr().getClient().commit(Const.SOLR_CORE_ATH_DOCS);
1752
1753
1754 if (isTempFile && fileToImport != null && fileToImport.exists()) {
1755 try {
1756 Files.deleteIfExists(fileToImport.toPath());
1757 Log.info(getClass(), "Deleted temp file: {}", fileToImport.getAbsolutePath());
1758
1759 } catch (Exception ex) {
1760 Log.warn(getClass(), "Failed to delete temp file: {}", ex.getMessage());
1761 }
1762 }
1763
1764 return Response.success(201, "Document uploaded successfully");
1765
1766 } catch (Exception e) {
1767 return Response.error(500, e, "Error uploading document");
1768
1769 } finally {
1770 if (isTempFile && fileToImport != null && fileToImport.exists()) {
1771 try {
1772 Files.deleteIfExists(fileToImport.toPath());
1773
1774 } catch (Exception ignored) {
1775 }
1776 }
1777 }
1778 }
1779
1780 public ResponseContext alignDocument(RequestContext request, UUID docId, JsonNode bodyNode) {
1781
1782 if (!ControllerUtil.checkParam(docId)) {
1783 return Response.error(400, "Invalid request parameter Doc Id: " + docId);
1784 }
1785
1786 if (!ControllerUtil.checkParam(bodyNode)) {
1787 return Response.error(400, "Invalid request parameter, bodyNode is null");
1788 }
1789
1790 AlignDocumentRequest body = AthUtil.safeFromJsonNode(bodyNode,
1791 AlignDocumentRequest.class, null);
1792
1793 if (body == null) {
1794 return Response.error(400, "Invalid request body");
1795 }
1796
1797 String srcLang = body.getSrcLang();
1798 String trgLang = body.getTrgLang();
1799 URI docTrlGcsUrl = body.getDocTrlGcsUrl();
1800 String docTrlEncoding = body.getDocTrlEncoding();
1801 String srcSrx = body.getSrcSrx();
1802 String trgSrx = body.getTrgSrx();
1803 Boolean useAlignmentModel = body.getUseAlignmentModel();
1804 String alignmentModelName = body.getAlignmentModelName();
1805 Boolean useCodesReinsertionModel = body.getUseCodesReinsertionModel();
1806 String codesReinsertionModelName = body.getCodesReinsertionModelName();
1807 UUID userId = body.getUserId();
1808
1809 if (!ControllerUtil.checkParam(srcLang)) {
1810 return Response.error(400, "Invalid request, srcLang is not specified");
1811 }
1812
1813 if (!ControllerUtil.checkParam(trgLang)) {
1814 return Response.error(400, "Invalid request, trgLang is not specified");
1815 }
1816
1817 if (!ControllerUtil.checkParam(docTrlGcsUrl)) {
1818 return Response.error(400, "Invalid request, doc_trl_gcs_url is not specified");
1819 }
1820
1821 if (!ControllerUtil.checkParam(userId)) {
1822 return Response.error(400, "Invalid request parameter User Id: " + userId);
1823 }
1824
1825 if (!AthStorage.exists(docTrlGcsUrl)) {
1826 return Response.error(404, "Translation object not found in GCS: " + docTrlGcsUrl);
1827 }
1828
1829 String query = Log.format("docId:\"{}\"", docId);
1830
1831 try {
1832 QueryResponse response = AthIndex.getMany(Const.SOLR_CORE_ATH_DOCS, query, null,
1833 QueryResponse.class);
1834
1835 if (response.getResults().isEmpty()) {
1836 return Response.error(404, "Document not found, docId: " + docId);
1837 }
1838
1839 SolrDocument existingDoc = response.getResults().get(0);
1840 String status = SolrUtil.safeGetField(existingDoc, Const.ATH_PROP_STATUS, null);
1841
1842 if (DocumentStatus.IMPORTING.toString().equals(status)
1843 || DocumentStatus.EXPORTING.toString().equals(status)) {
1844
1845 ConflictResponse conflict = new ConflictResponse();
1846 conflict.setError("Processing already in progress");
1847 conflict.setStatus(AthUtil.safeToEnum(status, DocumentStatus.class, null));
1848 conflict.setDocId(docId);
1849 conflict.setStatusUrl(Log.format("/document/{}/status", docId));
1850
1851 return Response.builder()
1852 .status(Status.CONFLICT)
1853 .header("Location", Log.format("/document/{}/status", docId))
1854 .entity(conflict)
1855 .build();
1856 }
1857
1858
1859 String docFileName = SolrUtil.safeGetField(existingDoc, Const.ATH_PROP_DOC_FILE_NAME, null);
1860
1861 URI docGcsUrl = AthUtil
1862 .toURI(SolrUtil.safeGetField(existingDoc, Const.ATH_PROP_DOC_STORAGE_NAME, null));
1863
1864 String docEncoding = SolrUtil.safeGetField(existingDoc, Const.ATH_PROP_DOC_FILE_ENCODING,
1865 null);
1866
1867 String filterId = SolrUtil.safeGetField(existingDoc, Const.ATH_PROP_FILTER_ID, null);
1868 String filterParams = SolrUtil.safeGetField(existingDoc, Const.ATH_PROP_FILTER_PARAMS, null);
1869
1870
1871 if (Util.isEmpty(filterId)) {
1872 String fileExt = Util.getExtension(docFileName);
1873 filterId = ControllerUtil.getFilterId(fileExt);
1874
1875 if (Util.isEmpty(filterId)) {
1876 return Response.error(400, "Cannot determine filter for file: " + docFileName);
1877 }
1878 }
1879
1880
1881 if (Util.isEmpty(docTrlEncoding)) {
1882 docTrlEncoding = docEncoding;
1883 }
1884
1885
1886 SolrInputDocument doc = SolrUtil.toInputDocument(existingDoc);
1887
1888 doc.setField(Const.ATH_PROP_SRC_LANG, srcLang);
1889 doc.setField(Const.ATH_PROP_TRG_LANG, trgLang);
1890 doc.setField(Const.ATH_PROP_DOC_TRL_STORAGE_NAME, docTrlGcsUrl.toString());
1891 doc.setField(Const.ATH_PROP_DOC_TRL_FILE_ENCODING, docTrlEncoding);
1892
1893 SolrUtil.safeSetField(doc, Const.ATH_PROP_FILTER_ID, filterId);
1894 SolrUtil.safeSetField(doc, Const.ATH_PROP_FILTER_PARAMS, filterParams);
1895 SolrUtil.safeSetField(doc, Const.ATH_PROP_SRC_SRX, srcSrx);
1896 SolrUtil.safeSetField(doc, Const.ATH_PROP_TRG_SRX, trgSrx);
1897
1898 doc.setField(Const.ATH_PROP_STATUS, DocumentStatus.IMPORTING.toString());
1899 doc.setField(Const.ATH_PROP_PROCESSED_BY, userId.toString());
1900 doc.setField(Const.ATH_PROP_STARTED_AT, new Date());
1901 doc.setField(Const.ATH_PROP_FINISHED_AT, null);
1902 doc.setField(Const.ATH_PROP_UPDATED_BY, userId.toString());
1903 doc.setField(Const.ATH_PROP_UPDATED_AT, new Date());
1904
1905 AthIndex.getSolr().getClient().add(Const.SOLR_CORE_ATH_DOCS, doc);
1906 AthIndex.getSolr().getClient().commit(Const.SOLR_CORE_ATH_DOCS);
1907
1908 final String docTrlEncodingRef = docTrlEncoding;
1909 final String filterIdRef = filterId;
1910
1911
1912 EXECUTOR.submit(() -> {
1913 ResponseContext res = Response.success(200);
1914
1915 try {
1916
1917 if (ControllerUtil.isCancelled(docId)) {
1918 Log.info(getClass(), "Alignment cancelled before starting for docId: {}", docId);
1919 return false;
1920 }
1921
1922 res = ControllerUtil.alignFile(
1923 doc,
1924 docId,
1925 null,
1926 docFileName,
1927 docGcsUrl,
1928 docEncoding,
1929 docTrlGcsUrl,
1930 docTrlEncodingRef,
1931 srcLang,
1932 trgLang,
1933 filterIdRef,
1934 filterParams,
1935 srcSrx,
1936 trgSrx,
1937 true,
1938 useAlignmentModel != null ? useAlignmentModel : false,
1939 alignmentModelName,
1940 useCodesReinsertionModel != null ? useCodesReinsertionModel : false,
1941 codesReinsertionModelName,
1942 userId);
1943
1944
1945 if (ControllerUtil.isCancelled(docId) || res.getStatus() == 499) {
1946 doc.setField(Const.ATH_PROP_STATUS, DocumentStatus.CANCELLED.toString());
1947
1948 } else if (res.getStatus() == 200) {
1949 doc.setField(Const.ATH_PROP_STATUS, DocumentStatus.IMPORT_COMPLETED.toString());
1950
1951 } else {
1952 doc.setField(Const.ATH_PROP_STATUS, DocumentStatus.FAILED.toString());
1953 doc.setField(Const.ATH_PROP_ERROR_MESSAGE, Response.getMessage(res));
1954 }
1955
1956 } catch (Exception e) {
1957 Log.error(getClass(), e, "Alignment failed");
1958
1959
1960 if (ControllerUtil.isCancelled(docId) || res.getStatus() == 499) {
1961 doc.setField(Const.ATH_PROP_STATUS, DocumentStatus.CANCELLED.toString());
1962
1963 } else {
1964 doc.setField(Const.ATH_PROP_STATUS, DocumentStatus.FAILED.toString());
1965 doc.setField(Const.ATH_PROP_ERROR_MESSAGE, e.getMessage());
1966 }
1967
1968 res = Response.error(500, e, "Alignment error");
1969
1970 } finally {
1971 doc.setField(Const.ATH_PROP_FINISHED_AT, new Date());
1972
1973 try {
1974 AthIndex.getSolr().getClient().add(Const.SOLR_CORE_ATH_DOCS, doc);
1975 AthIndex.getSolr().getClient().commit(Const.SOLR_CORE_ATH_DOCS);
1976
1977 } catch (Exception e) {
1978 Log.error(getClass(), e, "Failed to update Solr after alignment");
1979 }
1980 }
1981
1982 return res.getStatus() == 200 || res.getStatus() == 499;
1983 });
1984
1985 ProcessingResponse processingResponse = new ProcessingResponse();
1986 processingResponse.setStatus(ProcessingResponse.StatusEnum.IMPORTING);
1987 processingResponse.setDocId(docId);
1988 processingResponse.setStatusUrl(Log.format("/document/{}/status", docId));
1989 processingResponse.setSubmittedAt(new Date());
1990
1991 return Response.builder()
1992 .status(Status.ACCEPTED)
1993 .header("Location", Log.format("/document/{}/status", docId))
1994 .entity(processingResponse)
1995 .build();
1996
1997 } catch (Exception e) {
1998 return Response.error(500, "Error aligning document -- " + e.getMessage());
1999 }
2000 }
2001
2002 public ResponseContext processDocument(RequestContext request, UUID docId, JsonNode bodyNode) {
2003 if (!ControllerUtil.checkParam(docId)) {
2004 return Response.error(400, "Invalid request parameter Doc Id: " + docId);
2005 }
2006
2007 if (!ControllerUtil.checkParam(bodyNode)) {
2008 return Response.error(400, "Invalid request body");
2009 }
2010
2011 ProcessDocumentRequest body = AthUtil.safeFromJsonNode(bodyNode,
2012 ProcessDocumentRequest.class, null);
2013
2014 if (body == null) {
2015 return Response.error(400, "Invalid request body");
2016 }
2017
2018 String srcLang = body.getSrcLang();
2019 String trgLang = body.getTrgLang();
2020 String filterId = body.getFilterId();
2021 String filterParams = body.getFilterParams();
2022 String srcSrx = body.getSrcSrx();
2023 UUID tmId = body.getTmId();
2024 Integer tmThreshold = body.getTmThreshold();
2025 String mtEngineId = body.getMtEngineId();
2026 String mtEngineParams = body.getMtEngineParams();
2027
2028 List<MtResources> mtCustomResources = body.getMtCustomResources();
2029
2030 Boolean mtProvideConfidenceScores = body.getMtProvideConfidenceScores();
2031 Boolean mtUseTranslateLlm = body.getMtUseTranslateLlm();
2032 Boolean mtSendPlainText = body.getMtSendPlainText();
2033 Boolean useCodesReinsertionModel = body.getUseCodesReinsertionModel();
2034 String codesReinsertionModelName = body.getCodesReinsertionModelName();
2035 UUID userId = body.getUserId();
2036
2037 URI docGcsUrl = body.getDocGcsUrl();
2038 URI docOutGcsUrl = body.getDocOutGcsUrl();
2039 String docFileName = body.getDocFileName();
2040 String docEncoding = body.getDocEncoding();
2041 String docOutEncoding = body.getDocOutEncoding();
2042 Boolean adjustTargetPages = body.getAdjustTargetPages();
2043
2044
2045 if (!ControllerUtil.checkParam(docGcsUrl) || !ControllerUtil.checkParam(docOutGcsUrl)
2046 || !ControllerUtil.checkParam(docFileName) || !ControllerUtil.checkParam(userId)) {
2047 return Response.error(400,
2048 "Missing required fields: doc_gcs_url, doc_out_gcs_url, doc_file_name, or user_id");
2049 }
2050
2051 if (!AthStorage.exists(docGcsUrl)) {
2052 return Response.error(404, "Source document not found in GCS: " + docGcsUrl);
2053 }
2054
2055 try {
2056 String query = Log.format("docId:\"{}\"", docId);
2057
2058 QueryResponse response = AthIndex.getMany(Const.SOLR_CORE_ATH_DOCS, query, null,
2059 QueryResponse.class);
2060
2061 SolrDocument existingDoc = response.getResults().isEmpty() ? null
2062 : response.getResults().get(0);
2063
2064 String status = existingDoc != null
2065 ? SolrUtil.safeGetField(existingDoc, Const.ATH_PROP_STATUS, null)
2066 : null;
2067
2068 if (existingDoc != null && (DocumentStatus.IMPORTING.toString().equals(status)
2069 || DocumentStatus.EXPORTING.toString().equals(status))) {
2070 ConflictResponse conflict = new ConflictResponse()
2071 .error("Processing already in progress")
2072 .status(AthUtil.safeToEnum(status, DocumentStatus.class, null))
2073 .docId(docId)
2074 .statusUrl(Log.format("/document/{}/status", docId));
2075
2076 return Response.builder()
2077 .status(Status.CONFLICT)
2078 .header("Location", Log.format("/document/{}/status", docId))
2079 .entity(conflict)
2080 .build();
2081 }
2082
2083
2084 SolrInputDocument doc = existingDoc == null ? new SolrInputDocument()
2085 : SolrUtil.toInputDocument(existingDoc);
2086
2087 doc.setField(Const.ATH_PROP_DOC_ID, docId.toString());
2088 doc.setField(Const.ATH_PROP_CAT_FRAMEWORK_NAME, Const.CAT_FRAMEWORK_NAME);
2089 doc.setField(Const.ATH_PROP_CAT_FRAMEWORK_VERSION, Const.CAT_FRAMEWORK_VERSION);
2090 doc.setField(Const.ATH_PROP_DOC_FILE_NAME, docFileName);
2091 doc.setField(Const.ATH_PROP_DOC_STORAGE_NAME, docGcsUrl.toString());
2092 doc.setField(Const.ATH_PROP_DOC_FILE_ENCODING, docEncoding);
2093 doc.setField(Const.ATH_PROP_DOC_OUT_STORAGE_NAME, docOutGcsUrl.toString());
2094 doc.setField(Const.ATH_PROP_DOC_OUT_FILE_ENCODING, docOutEncoding);
2095
2096 doc.setField(Const.ATH_PROP_SRC_LANG, srcLang);
2097 doc.setField(Const.ATH_PROP_TRG_LANG, trgLang);
2098 SolrUtil.safeSetField(doc, Const.ATH_PROP_FILTER_ID, filterId);
2099 SolrUtil.safeSetField(doc, Const.ATH_PROP_FILTER_PARAMS, filterParams);
2100 SolrUtil.safeSetField(doc, Const.ATH_PROP_SRC_SRX, srcSrx);
2101 SolrUtil.safeSetField(doc, Const.ATH_PROP_TM_ID, tmId);
2102 doc.setField(Const.ATH_PROP_TM_THRESHOLD, tmThreshold);
2103 SolrUtil.safeSetField(doc, Const.ATH_PROP_MT_ENGINE_ID, mtEngineId);
2104 SolrUtil.safeSetField(doc, Const.ATH_PROP_MT_ENGINE_PARAMS, mtEngineParams);
2105
2106 doc.setField(Const.ATH_PROP_STATUS, DocumentStatus.IMPORTING.toString());
2107 doc.setField(Const.ATH_PROP_PROCESSED_BY, userId.toString());
2108 doc.setField(Const.ATH_PROP_STARTED_AT, new Date());
2109 doc.setField(Const.ATH_PROP_UPDATED_BY, userId.toString());
2110 doc.setField(Const.ATH_PROP_UPDATED_AT, new Date());
2111
2112 AthIndex.getSolr().getClient().add(Const.SOLR_CORE_ATH_DOCS, doc);
2113 AthIndex.getSolr().getClient().commit(Const.SOLR_CORE_ATH_DOCS);
2114
2115
2116 EXECUTOR.submit(() -> {
2117 SolrInputDocument currentDoc = doc;
2118 boolean importSuccess = false;
2119
2120 try {
2121
2122 if (ControllerUtil.isCancelled(docId)) {
2123 Log.info(getClass(), "Process cancelled before starting for docId: {}", docId);
2124 return;
2125 }
2126
2127
2128 ResponseContext importRes = ControllerUtil.processFileImportPhase(
2129 currentDoc, docId, docFileName, docGcsUrl, docEncoding,
2130 srcLang, trgLang, filterId, filterParams, srcSrx,
2131 tmId, tmThreshold, mtEngineId, mtEngineParams, mtCustomResources,
2132 mtProvideConfidenceScores, mtUseTranslateLlm,
2133 mtSendPlainText, useCodesReinsertionModel, codesReinsertionModelName, userId,
2134 existingDoc == null);
2135
2136 importSuccess = importRes.getStatus() == 200;
2137
2138
2139 if (ControllerUtil.isCancelled(docId) || importRes.getStatus() == 499) {
2140 doc.setField(Const.ATH_PROP_STATUS, DocumentStatus.CANCELLED.toString());
2141
2142 } else if (importSuccess) {
2143
2144 currentDoc.setField(Const.ATH_PROP_STATUS, DocumentStatus.EXPORTING.toString());
2145 currentDoc.setField(Const.ATH_PROP_UPDATED_AT, new Date());
2146 AthIndex.getSolr().getClient().add(Const.SOLR_CORE_ATH_DOCS, currentDoc);
2147 AthIndex.getSolr().getClient().commit(Const.SOLR_CORE_ATH_DOCS);
2148
2149
2150 SolrDocument finalDoc = SolrUtil.getDocumentByDocId(docId);
2151 SolrDocument tmDoc = tmId != null ? SolrUtil.getTmByTmId(tmId) : null;
2152
2153 ResponseContext exportRes = ControllerUtil.exportFile(
2154 finalDoc, tmDoc, currentDoc, docOutGcsUrl, docOutEncoding, tmId,
2155 adjustTargetPages != null ? adjustTargetPages : false, userId);
2156
2157
2158 if (ControllerUtil.isCancelled(docId) || exportRes.getStatus() == 499) {
2159 doc.setField(Const.ATH_PROP_STATUS, DocumentStatus.CANCELLED.toString());
2160
2161 } else if (exportRes.getStatus() == 200) {
2162 currentDoc.setField(Const.ATH_PROP_STATUS,
2163 DocumentStatus.EXPORT_COMPLETED.toString());
2164
2165 } else {
2166 currentDoc.setField(Const.ATH_PROP_STATUS, DocumentStatus.FAILED.toString());
2167 currentDoc.setField(Const.ATH_PROP_ERROR_MESSAGE, Response.getMessage(exportRes));
2168 }
2169
2170 } else {
2171 currentDoc.setField(Const.ATH_PROP_STATUS, DocumentStatus.FAILED.toString());
2172 currentDoc.setField(Const.ATH_PROP_ERROR_MESSAGE, Response.getMessage(importRes));
2173 }
2174
2175 } catch (Exception e) {
2176 Log.error(getClass(), e, "Process failed for docId: {}", docId);
2177
2178
2179 if (ControllerUtil.isCancelled(docId)) {
2180 doc.setField(Const.ATH_PROP_STATUS, DocumentStatus.CANCELLED.toString());
2181
2182 } else {
2183 currentDoc.setField(Const.ATH_PROP_STATUS, DocumentStatus.FAILED.toString());
2184 currentDoc.setField(Const.ATH_PROP_ERROR_MESSAGE, e.getMessage());
2185 }
2186
2187 } finally {
2188 currentDoc.setField(Const.ATH_PROP_FINISHED_AT, new Date());
2189 currentDoc.setField(Const.ATH_PROP_UPDATED_BY, userId.toString());
2190 currentDoc.setField(Const.ATH_PROP_UPDATED_AT, new Date());
2191
2192 try {
2193 AthIndex.getSolr().getClient().add(Const.SOLR_CORE_ATH_DOCS, currentDoc);
2194 AthIndex.getSolr().getClient().commit(Const.SOLR_CORE_ATH_DOCS);
2195
2196 } catch (Exception e) {
2197 Log.error(getClass(), e, "Failed to update status after process");
2198 }
2199 }
2200 });
2201
2202 ProcessingResponse processingResponse = new ProcessingResponse()
2203 .status(ProcessingResponse.StatusEnum.IMPORTING)
2204 .docId(docId)
2205 .statusUrl(Log.format("/document/{}/status", docId))
2206 .submittedAt(new Date());
2207
2208 return Response.builder()
2209 .status(Status.ACCEPTED)
2210 .header("Location", Log.format("/document/{}/status", docId))
2211 .entity(processingResponse)
2212 .build();
2213
2214 } catch (Exception e) {
2215 return Response.error(500, e, "Error starting document processing");
2216 }
2217 }
2218 }