1 package com.acumenvelocity.ath.controller;
2
3 import java.io.File;
4 import java.nio.charset.StandardCharsets;
5 import java.util.ArrayList;
6 import java.util.Date;
7 import java.util.List;
8 import java.util.Objects;
9 import java.util.UUID;
10 import java.util.stream.Collectors;
11
12 import javax.ws.rs.core.Response.Status;
13
14 import org.apache.commons.lang3.math.NumberUtils;
15 import org.apache.solr.client.solrj.SolrClient;
16 import org.apache.solr.client.solrj.SolrQuery;
17 import org.apache.solr.client.solrj.response.QueryResponse;
18 import org.apache.solr.common.SolrDocument;
19 import org.apache.solr.common.SolrDocumentList;
20 import org.apache.solr.common.SolrInputDocument;
21
22 import com.acumenvelocity.ath.common.AthUtil;
23 import com.acumenvelocity.ath.common.Const;
24 import com.acumenvelocity.ath.common.ControllerUtil;
25 import com.acumenvelocity.ath.common.JacksonUtil;
26 import com.acumenvelocity.ath.common.Log;
27 import com.acumenvelocity.ath.common.Response;
28 import com.acumenvelocity.ath.common.SolrUtil;
29 import com.acumenvelocity.ath.common.exception.AthException;
30 import com.acumenvelocity.ath.gcs.AthStorage;
31 import com.acumenvelocity.ath.model.CreateTranslationMemorySegmentRequest;
32 import com.acumenvelocity.ath.model.PaginationInfo;
33 import com.acumenvelocity.ath.model.TmSegment;
34 import com.acumenvelocity.ath.model.TmSegmentWrapper;
35 import com.acumenvelocity.ath.model.TmSegmentsWrapper;
36 import com.acumenvelocity.ath.model.TranslationMemoryInfo;
37 import com.acumenvelocity.ath.model.TranslationMemoryInfoWrapper;
38 import com.acumenvelocity.ath.model.TranslationMemoryInfosWrapper;
39 import com.acumenvelocity.ath.model.UpdateTranslationMemorySegmentRequest;
40 import com.acumenvelocity.ath.model.x.LayeredTextX;
41 import com.acumenvelocity.ath.solr.AthIndex;
42 import com.acumenvelocity.ath.solr.tm.SolrTmFilter;
43 import com.acumenvelocity.ath.solr.tm.SolrTmWriterStep;
44 import com.fasterxml.jackson.databind.JsonNode;
45
46 import io.swagger.oas.inflector.models.RequestContext;
47 import io.swagger.oas.inflector.models.ResponseContext;
48 import net.sf.okapi.common.LocaleId;
49 import net.sf.okapi.common.Util;
50 import net.sf.okapi.common.filters.FilterUtil;
51 import net.sf.okapi.common.filters.IFilter;
52 import net.sf.okapi.common.filters.WrapMode;
53 import net.sf.okapi.common.pipelinebuilder.XBatch;
54 import net.sf.okapi.common.pipelinebuilder.XBatchItem;
55 import net.sf.okapi.common.pipelinebuilder.XParameter;
56 import net.sf.okapi.common.pipelinebuilder.XPipeline;
57 import net.sf.okapi.common.pipelinebuilder.XPipelineStep;
58 import net.sf.okapi.common.resource.ITextUnit;
59 import net.sf.okapi.common.resource.RawDocument;
60 import net.sf.okapi.common.resource.TextUnit;
61 import net.sf.okapi.filters.table.csv.CommaSeparatedValuesFilter;
62 import net.sf.okapi.filters.table.tsv.TabSeparatedValuesFilter;
63 import net.sf.okapi.steps.common.RawDocumentToFilterEventsStep;
64 import net.sf.okapi.steps.formatconversion.FormatConversionStep;
65 import net.sf.okapi.steps.formatconversion.Parameters;
66
67 public class TranslationMemoryController {
68 private static final String TMX_FILE_NAME = "download.tmx";
69
70 public ResponseContext getTranslationMemories(RequestContext request, Integer page,
71 Integer pageSize, UUID companyId) {
72 try {
73 SolrClient solrClient = AthIndex.getSolr().getClient();
74
75
76 SolrQuery q = new SolrQuery("*:*")
77 .setFields(Const.ATH_PROP_TM_ID, Const.ATH_PROP_TM_COMPANY_ID)
78 .addSort("_docid_", SolrQuery.ORDER.asc)
79 .setRows(Integer.MAX_VALUE);
80
81
82 if (companyId != null) {
83 q.setQuery("{!term f=" + Const.ATH_PROP_TM_COMPANY_ID + "}" + companyId.toString());
84 }
85
86 QueryResponse response = solrClient.query(Const.SOLR_CORE_ATH_TMS, q);
87
88 List<String> allTmIds = response.getResults().stream()
89 .map(doc -> doc.getFieldValue(Const.ATH_PROP_TM_ID))
90 .filter(Objects::nonNull)
91 .map(Object::toString)
92 .filter(id -> !id.isEmpty())
93 .distinct()
94 .collect(Collectors.toList());
95
96 TranslationMemoryInfosWrapper wrapper = new TranslationMemoryInfosWrapper();
97
98
99 if (allTmIds.isEmpty()) {
100 wrapper.translationMemories(new ArrayList<>())
101 .pagination(new PaginationInfo()
102 .page(1)
103 .pageSize(0)
104 .totalItems(0L)
105 .totalPages(0)
106 .hasNext(false)
107 .hasPrevious(false));
108
109 return Response.success(200, wrapper);
110 }
111
112 long totalItems = allTmIds.size();
113 int size = (pageSize != null) ? Math.max(1, Math.min(100, pageSize)) : (int) totalItems;
114 int totalPages = (int) Math.ceil(totalItems / (double) size);
115
116 int pageNum;
117 List<String> tmIdsToProcess;
118
119 if (page == null && pageSize == null) {
120
121 pageNum = 1;
122 int end = Math.min(size, allTmIds.size());
123 tmIdsToProcess = allTmIds.subList(0, end);
124
125 } else {
126 pageNum = (page != null) ? Math.max(1, Math.min(page, Math.max(1, totalPages))) : 1;
127 int start = (pageNum - 1) * size;
128 int end = Math.min(start + size, allTmIds.size());
129 tmIdsToProcess = allTmIds.subList(start, end);
130 }
131
132 List<TranslationMemoryInfo> resultTmInfos = tmIdsToProcess.stream()
133 .map(this::getTranslationMemoryInfoInternal)
134 .filter(Objects::nonNull)
135 .collect(Collectors.toList());
136
137 PaginationInfo pagination = new PaginationInfo()
138 .page(pageNum)
139 .pageSize(size)
140 .totalItems(totalItems)
141 .totalPages(totalPages)
142 .hasNext(pageNum < totalPages)
143 .hasPrevious(pageNum > 1);
144
145 wrapper.translationMemories(resultTmInfos)
146 .pagination(pagination);
147
148 return Response.success(200, wrapper);
149
150 } catch (Exception e) {
151 return Response.error(500, e, "Error fetching translation memories");
152 }
153 }
154
155 private TranslationMemoryInfo getTranslationMemoryInfoInternal(String tmId) {
156 try {
157 SolrClient solrClient = AthIndex.getSolr().getClient();
158 String query = Log.format("tmId:\"{}\"", tmId);
159 SolrQuery solrQuery = new SolrQuery(query);
160 solrQuery.setRows(1);
161 solrQuery.setFields(
162 Const.ATH_PROP_TM_COMPANY_ID,
163 Const.ATH_PROP_TM_ID,
164 Const.ATH_PROP_TM_STORAGE_NAME,
165 Const.ATH_PROP_TM_FILE_NAME,
166 Const.ATH_PROP_SRC_LANG,
167 Const.ATH_PROP_TRG_LANG,
168 Const.ATH_PROP_CREATED_BY,
169 Const.ATH_PROP_CREATED_AT,
170 Const.ATH_PROP_UPDATED_BY,
171 Const.ATH_PROP_UPDATED_AT);
172
173 QueryResponse response = solrClient.query(Const.SOLR_CORE_ATH_TMS, solrQuery);
174 SolrDocumentList docList = response.getResults();
175
176 if (docList.isEmpty()) {
177 return null;
178 }
179
180 SolrDocument doc = docList.get(0);
181 TranslationMemoryInfo tmInfo = new TranslationMemoryInfo();
182
183 tmInfo.setTmId(
184 AthUtil.safeToUuid(SolrUtil.safeGetField(doc, Const.ATH_PROP_TM_ID, null), null));
185
186 tmInfo.setTmCompanyId(
187 AthUtil.safeToUuid(SolrUtil.safeGetField(doc, Const.ATH_PROP_TM_COMPANY_ID, null), null));
188
189 tmInfo.setTmGcsUrl(
190 AthUtil.toURI(SolrUtil.safeGetField(doc, Const.ATH_PROP_TM_STORAGE_NAME, null)));
191
192 tmInfo.setTmFileName(SolrUtil.safeGetField(doc, Const.ATH_PROP_TM_FILE_NAME, null));
193 tmInfo.setSrcLang(SolrUtil.safeGetField(doc, Const.ATH_PROP_SRC_LANG, null));
194 tmInfo.setTrgLang(SolrUtil.safeGetField(doc, Const.ATH_PROP_TRG_LANG, null));
195
196 tmInfo.setCreatedBy(
197 AthUtil.safeToUuid(SolrUtil.safeGetField(doc, Const.ATH_PROP_CREATED_BY, null), null));
198
199 tmInfo.setCreatedAt(AthUtil.safeToDate(doc.get(Const.ATH_PROP_CREATED_AT), null));
200
201 tmInfo.setUpdatedBy(
202 AthUtil.safeToUuid(SolrUtil.safeGetField(doc, Const.ATH_PROP_UPDATED_BY, null), null));
203
204 tmInfo.setUpdatedAt(AthUtil.safeToDate(doc.get(Const.ATH_PROP_UPDATED_AT), null));
205
206
207 long segmentsCount = SolrUtil.getNumDocuments(Const.SOLR_CORE_ATH_TM_SEGMENTS, query);
208 tmInfo.setSegmentsCount(segmentsCount);
209
210 return tmInfo;
211
212 } catch (Exception e) {
213 return null;
214 }
215 }
216
217 public ResponseContext createTranslationMemory(RequestContext request, UUID companyId, UUID tmId,
218 String tmGcsUrl, String tmFileName, Integer tmFileStartLine, String tmFileDelimiter,
219 Integer tmFileSrcColumn, Integer tmFileTrgColumn, String tmSrcLang, String tmTrgLang,
220 UUID userId) {
221
222 if (!ControllerUtil.checkParam(tmId)) {
223 return Response.error(400, "Invalid request parameter TM Id: " + tmId);
224 }
225
226 if (!ControllerUtil.checkParam(tmGcsUrl)) {
227 return Response.error(400, "Invalid request, tmGcsUrl is not specified");
228 }
229
230 if (!ControllerUtil.checkParam(tmFileName)) {
231 return Response.error(400, "Invalid request, tmFileName is not specified");
232 }
233
234 if (!ControllerUtil.checkParam(tmSrcLang)) {
235 return Response.error(400, "Invalid request, tmSrcLang is not specified");
236 }
237
238 if (!ControllerUtil.checkParam(tmTrgLang)) {
239 return Response.error(400, "Invalid request, tmTrgLang is not specified");
240 }
241
242 if (!ControllerUtil.checkParam(userId)) {
243 return Response.error(400, "Invalid request parameter User Id: " + userId);
244 }
245
246 tmFileName = Util.getFilename(tmGcsUrl, true);
247 File tmFile = null;
248
249 try {
250 tmFile = AthUtil.createTempFile();
251
252
253 AthStorage.loadFile(AthUtil.toURI(tmGcsUrl), tmFile);
254
255 } catch (Exception e) {
256 return Response.error(500, "Cannot create temp TM file -- " + e.getMessage());
257 }
258
259
260 String query = Log.format("tmId:\"{}\"", tmId);
261
262 try {
263 if (SolrUtil.getNumDocuments(Const.SOLR_CORE_ATH_TMS, query) > 0) {
264 return Response.error(400, "Translation Memory already exists, tmId: " + tmId);
265 }
266
267 } catch (Exception e) {
268 return Response.error(500, "Error checking TM existence -- " + e.getMessage());
269 }
270
271 LocaleId srcLoc = LocaleId.fromString(tmSrcLang);
272 LocaleId trgLoc = LocaleId.fromString(tmTrgLang);
273
274 IFilter filter = getFilter(tmFileName, tmFileStartLine, tmFileDelimiter, tmFileSrcColumn,
275 tmFileTrgColumn);
276
277 try {
278
279 SolrInputDocument tmDoc = new SolrInputDocument();
280
281 if (companyId != null) {
282 tmDoc.addField(Const.ATH_PROP_TM_COMPANY_ID, companyId.toString());
283 }
284
285 tmDoc.addField(Const.ATH_PROP_TM_ID, tmId.toString());
286 tmDoc.addField(Const.ATH_PROP_TM_FILE_NAME, tmFileName);
287 tmDoc.addField(Const.ATH_PROP_TM_STORAGE_NAME, tmGcsUrl);
288 tmDoc.addField(Const.ATH_PROP_SRC_LANG, tmSrcLang);
289 tmDoc.addField(Const.ATH_PROP_TRG_LANG, tmTrgLang);
290
291 if (userId != null) {
292 tmDoc.addField(Const.ATH_PROP_CREATED_BY, userId.toString());
293 tmDoc.addField(Const.ATH_PROP_UPDATED_BY, userId.toString());
294 }
295
296 tmDoc.addField(Const.ATH_PROP_CREATED_AT, new Date());
297 tmDoc.addField(Const.ATH_PROP_UPDATED_AT, new Date());
298
299 SolrClient solrClient = AthIndex.getSolr().getClient();
300
301 solrClient.add(Const.SOLR_CORE_ATH_TMS, tmDoc);
302 solrClient.commit(Const.SOLR_CORE_ATH_TMS);
303
304
305 try (XPipeline pl = new XPipeline("TM export",
306 new XBatch(
307 new XBatchItem(
308 Util.toURI(tmFile.getAbsolutePath()),
309 StandardCharsets.UTF_8.name(),
310 srcLoc, trgLoc)),
311 new RawDocumentToFilterEventsStep(filter),
312 new SolrTmWriterStep(tmId, tmFileName, userId, true))) {
313
314 pl.execute();
315
316 } catch (Exception e) {
317
318 solrClient.deleteByQuery(Const.SOLR_CORE_ATH_TMS, query);
319 solrClient.commit(Const.SOLR_CORE_ATH_TMS);
320 throw e;
321 }
322
323 } catch (Exception e) {
324 return Response.error(500, "Error creating TM -- " + e.getMessage());
325
326 } finally {
327 AthUtil.deleteFile(tmFile);
328 }
329
330 return Response.success(201, "Success");
331 }
332
333 private IFilter getFilter(String tmFileName, Integer tmFileStartLine, String tmFileDelimiter,
334 Integer tmFileSrcColumn, Integer tmFileTrgColumn) {
335
336 String fileExt = com.google.common.io.Files.getFileExtension(tmFileName);
337
338 if ("tmx".equalsIgnoreCase(fileExt)) {
339 return FilterUtil.createFilter("okf_tmx");
340
341 } else {
342 net.sf.okapi.filters.table.csv.Parameters csvParams = new net.sf.okapi.filters.table.csv.Parameters();
343
344 csvParams.textQualifier = "\"";
345 csvParams.removeQualifiers = true;
346 csvParams.escapingMode = net.sf.okapi.filters.table.csv.Parameters.ESCAPING_MODE_DUPLICATION;
347 csvParams.addQualifiers = false;
348 csvParams.columnNamesLineNum = 0;
349 csvParams.valuesStartLineNum = tmFileStartLine;
350 csvParams.detectColumnsMode = net.sf.okapi.filters.table.csv.Parameters.DETECT_COLUMNS_NONE;
351 csvParams.numColumns = NumberUtils.max(tmFileSrcColumn, tmFileTrgColumn);
352 csvParams.sendHeaderMode = net.sf.okapi.filters.table.csv.Parameters.SEND_HEADER_NONE;
353 csvParams.trimMode = net.sf.okapi.filters.table.csv.Parameters.TRIM_NONQUALIFIED_ONLY;
354 csvParams.sendColumnsMode = net.sf.okapi.filters.table.csv.Parameters.SEND_COLUMNS_LISTED;
355 csvParams.sourceIdColumns = "";
356 csvParams.sourceColumns = tmFileSrcColumn == 0 ? "" : String.valueOf(tmFileSrcColumn);
357 csvParams.targetColumns = tmFileTrgColumn == 0 ? "" : String.valueOf(tmFileTrgColumn);
358 csvParams.commentColumns = "";
359 csvParams.commentSourceRefs = csvParams.sourceColumns;
360 csvParams.recordIdColumn = 0;
361 csvParams.sourceIdSourceRefs = "";
362 csvParams.sourceIdSuffixes = "";
363 csvParams.targetLanguages = "";
364 csvParams.targetSourceRefs = csvParams.sourceColumns;
365 csvParams.trimLeading = true;
366 csvParams.trimTrailing = true;
367 csvParams.preserveWS = true;
368 csvParams.useCodeFinder = false;
369 csvParams.wrapMode = WrapMode.NONE;
370 csvParams.subfilter = null;
371
372 if ("tsv".equalsIgnoreCase(fileExt)) {
373 csvParams.fieldDelimiter = Util.isEmpty(tmFileDelimiter) ? "\t" : tmFileDelimiter;
374
375 TabSeparatedValuesFilter tsvFilter = new TabSeparatedValuesFilter();
376 tsvFilter.setParameters(csvParams);
377 return tsvFilter;
378
379 } else {
380 csvParams.fieldDelimiter = Util.isEmpty(tmFileDelimiter) ? "," : tmFileDelimiter;
381
382 CommaSeparatedValuesFilter csvFilter = new CommaSeparatedValuesFilter();
383 csvFilter.setParameters(csvParams);
384 return csvFilter;
385 }
386 }
387 }
388
389 public ResponseContext exportTranslationMemory(RequestContext request, UUID tmId) {
390 if (!ControllerUtil.checkParam(tmId)) {
391 return Response.error(400, "Invalid request parameter TM Id: " + tmId);
392 }
393
394 String query = Log.format("tmId:\"{}\"", tmId);
395
396 try {
397 if (SolrUtil.getNumDocuments(Const.SOLR_CORE_ATH_TMS, query) <= 0) {
398 return Response.error(404, "TM not found, tmId: " + tmId);
399 }
400
401 File file = null;
402
403 try {
404 file = AthUtil.createTempFile();
405
406 try (SolrTmFilter tmFilter = new SolrTmFilter(AthIndex.getSolr().getClient(),
407 Const.SOLR_CORE_ATH_TM_SEGMENTS, tmId)) {
408
409 LocaleId srcLoc = null;
410 LocaleId trgLoc = null;
411
412 TranslationMemoryInfo tmInfo = getTranslationMemoryInfoInternal(tmId.toString());
413
414 try {
415 srcLoc = LocaleId.fromString(tmInfo.getSrcLang());
416 trgLoc = LocaleId.fromString(tmInfo.getTrgLang());
417
418 } catch (Exception e) {
419 Log.warn(getClass(), "Error detecting TM locales: tmId='{}' -- {}", tmId,
420 e.getMessage());
421 }
422
423 try (XPipeline pl = new XPipeline("TM export",
424 new XBatch(
425 new XBatchItem(
426 new RawDocument(
427 "{}",
428 srcLoc == null ? LocaleId.AUTODETECT : srcLoc,
429 trgLoc == null ? LocaleId.AUTODETECT : trgLoc))),
430
431 new RawDocumentToFilterEventsStep(tmFilter),
432
433 new XPipelineStep(FormatConversionStep.class,
434 new XParameter("outputFormat", Parameters.FORMAT_TMX),
435 new XParameter("outputPath", file.toURI().getPath()),
436 new XParameter("singleOutput", true),
437 new XParameter("overwriteSameSource", false)))) {
438
439 pl.execute();
440 }
441 }
442
443 } catch (Exception e) {
444 return Response.error(500, "Cannot create temp file -- " + e.getMessage());
445 }
446
447 return Response.builder()
448 .status(Status.OK)
449 .header("Content-Disposition", Log.format("attachment; filename=\"{}\"", TMX_FILE_NAME))
450 .entity(file)
451 .build();
452
453 } catch (Exception e) {
454 return Response.error(500, "Error exporting TM -- " + e.getMessage());
455 }
456 }
457
458 public ResponseContext updateTranslationMemory(RequestContext request, UUID tmId,
459 UUID companyId, String tmGcsUrl, String tmFileName, Integer tmFileStartLine,
460 String tmFileDelimiter, Integer tmFileSrcColumn, Integer tmFileTrgColumn, UUID userId) {
461
462 if (!ControllerUtil.checkParam(tmId)) {
463 return Response.error(400, "Invalid request parameter TM Id: " + tmId);
464 }
465
466 if (!ControllerUtil.checkParam(tmGcsUrl)) {
467 return Response.error(400, "Invalid request, tmGcsUrl is not specified");
468 }
469
470 if (!ControllerUtil.checkParam(tmFileName)) {
471 return Response.error(400, "Invalid request, tmFileName is not specified");
472 }
473
474 if (!ControllerUtil.checkParam(userId)) {
475 return Response.error(400, "Invalid request parameter User Id: " + userId);
476 }
477
478 tmFileName = Util.getFilename(tmGcsUrl, true);
479 File tmFile = null;
480
481 try {
482 tmFile = AthUtil.createTempFile();
483
484
485 AthStorage.loadFile(AthUtil.toURI(tmGcsUrl), tmFile);
486
487 } catch (Exception e) {
488 return Response.error(500, "Cannot create temp TM file -- " + e.getMessage());
489 }
490
491 String query = Log.format("tmId:\"{}\"", tmId);
492
493 try {
494 if (SolrUtil.getNumDocuments(Const.SOLR_CORE_ATH_TMS, query) <= 0) {
495 return Response.error(404, "TM not found, tmId: " + tmId);
496 }
497
498 LocaleId srcLoc = LocaleId.ENGLISH;
499 LocaleId trgLoc = LocaleId.FRENCH;
500
501
502 TranslationMemoryInfo tmInfo = getTranslationMemoryInfoInternal(tmId.toString());
503
504 if (tmInfo != null) {
505 srcLoc = LocaleId.fromString(tmInfo.getSrcLang());
506 trgLoc = LocaleId.fromString(tmInfo.getTrgLang());
507 }
508
509 IFilter filter = getFilter(tmFileName, tmFileStartLine, tmFileDelimiter, tmFileSrcColumn,
510 tmFileTrgColumn);
511
512 try (XPipeline pl = new XPipeline("TM update",
513 new XBatch(
514 new XBatchItem(
515 Util.toURI(tmFile.getAbsolutePath()),
516 StandardCharsets.UTF_8.name(),
517 srcLoc, trgLoc)),
518 new RawDocumentToFilterEventsStep(filter),
519 new SolrTmWriterStep(tmId, tmFileName, userId, false))) {
520
521 pl.execute();
522
523
524 SolrClient solrClient = AthIndex.getSolr().getClient();
525 SolrInputDocument updateDoc = SolrUtil.toInputDocument(SolrUtil.getTmByTmId(tmId));
526
527 updateDoc.setField(Const.ATH_PROP_TM_FILE_NAME, tmFileName);
528 updateDoc.setField(Const.ATH_PROP_TM_STORAGE_NAME, tmGcsUrl);
529 updateDoc.setField(Const.ATH_PROP_UPDATED_BY, userId.toString());
530 updateDoc.setField(Const.ATH_PROP_UPDATED_AT, new Date());
531
532 solrClient.add(Const.SOLR_CORE_ATH_TMS, updateDoc);
533 solrClient.commit(Const.SOLR_CORE_ATH_TMS);
534
535 } catch (Exception e) {
536 return Response.error(500, "Error updating TM -- " + e.getMessage());
537
538 } finally {
539 AthUtil.deleteFile(tmFile);
540 }
541
542 return Response.success(200, "Success");
543
544 } catch (Exception e) {
545 return Response.error(500, "Error updating TM -- " + e.getMessage());
546 }
547 }
548
549 public ResponseContext deleteTranslationMemory(RequestContext request, UUID tmId) {
550 if (!ControllerUtil.checkParam(tmId)) {
551 return Response.error(400, "Invalid request parameter TM Id: " + tmId);
552 }
553
554 String query = Log.format("tmId:\"{}\"", tmId);
555
556 try {
557 if (SolrUtil.getNumDocuments(Const.SOLR_CORE_ATH_TMS, query) <= 0) {
558 return Response.error(404, "TM not found, tmId: " + tmId);
559 }
560
561
562 AthIndex.deleteByQuery(Const.SOLR_CORE_ATH_TM_SEGMENTS, query);
563 AthIndex.deleteByQuery(Const.SOLR_CORE_ATH_TMS, query);
564
565 return Response.success(204, "Translation Memory (id={}) was deleted successfully", tmId);
566
567 } catch (Exception e) {
568 String st = Log.format("Error deleting Translation Memory (id={}) -- {}",
569 tmId, e.getMessage());
570
571 Log.error(this.getClass(), e, st);
572 return Response.error(500, st);
573 }
574 }
575
576 public ResponseContext getTranslationMemoryInfo(RequestContext request, UUID tmId) {
577 if (!ControllerUtil.checkParam(tmId)) {
578 return Response.error(400, "Invalid request parameter TM Id: " + tmId);
579 }
580
581 TranslationMemoryInfo tmInfo = getTranslationMemoryInfoInternal(tmId.toString());
582
583 if (tmInfo == null) {
584 return Response.error(404, "TM not found, tmId: " + tmId);
585 }
586
587 TranslationMemoryInfoWrapper tiw = new TranslationMemoryInfoWrapper();
588 tiw.setTranslationMemoryInfo(tmInfo);
589
590 return Response.success(200, tiw);
591 }
592
593 public ResponseContext getTranslationMemorySegments(RequestContext request, UUID tmId,
594 Integer page, Integer pageSize) {
595
596 if (!ControllerUtil.checkParam(tmId)) {
597 return Response.error(400, "Invalid request parameter TM Id: " + tmId);
598 }
599
600 String query = Log.format("tmId:\"{}\"", tmId);
601
602 try {
603 long totalItems = SolrUtil.getNumDocuments(Const.SOLR_CORE_ATH_TM_SEGMENTS, query);
604
605 if (totalItems <= 0) {
606 TmSegmentsWrapper emptyWrapper = new TmSegmentsWrapper()
607 .tmSegments(new ArrayList<>())
608 .pagination(new PaginationInfo()
609 .page(1)
610 .pageSize(0)
611 .totalItems(0L)
612 .totalPages(0)
613 .hasNext(false)
614 .hasPrevious(false));
615
616 return Response.success(200, emptyWrapper);
617 }
618
619 int size = (pageSize != null) ? Math.max(1, Math.min(100, pageSize)) : (int) totalItems;
620 int totalPages = (int) Math.ceil(totalItems / (double) size);
621 int pageNum = (page != null) ? Math.max(1, Math.min(page, Math.max(1, totalPages))) : 1;
622
623 SolrClient solrClient = AthIndex.getSolr().getClient();
624 SolrQuery solrQuery = new SolrQuery(query)
625 .setStart((pageNum - 1) * size)
626 .setRows(size)
627 .addSort(Const.ATH_PROP_CREATED_AT, SolrQuery.ORDER.asc);
628
629 QueryResponse response = solrClient.query(Const.SOLR_CORE_ATH_TM_SEGMENTS, solrQuery);
630 SolrDocumentList docList = response.getResults();
631
632 List<TmSegment> segments = docList.stream()
633 .map(this::toTmSegment)
634 .filter(Objects::nonNull)
635 .collect(Collectors.toList());
636
637
638 PaginationInfo pagination = new PaginationInfo()
639 .page(pageNum)
640 .pageSize(size)
641 .totalItems(totalItems)
642 .totalPages(totalPages)
643 .hasNext(pageNum < totalPages)
644 .hasPrevious(pageNum > 1);
645
646 TmSegmentsWrapper wrapper = new TmSegmentsWrapper()
647 .tmSegments(segments)
648 .pagination(pagination);
649
650 return Response.success(200, wrapper);
651
652 } catch (Exception e) {
653 return Response.error(500, e, "Error fetching TM segments");
654 }
655 }
656
657
658
659
660
661
662
663
664
665
666 public ResponseContext createTranslationMemorySegment(RequestContext request, UUID tmId,
667 JsonNode bodyNode) {
668
669 if (!ControllerUtil.checkParam(tmId)) {
670 return Response.error(400, "Invalid request parameter TM Id: " + tmId);
671 }
672
673 if (!ControllerUtil.checkParam(bodyNode)) {
674 return Response.error(400, "Invalid request body");
675 }
676
677 CreateTranslationMemorySegmentRequest body = AthUtil.safeFromJsonNode(bodyNode,
678 CreateTranslationMemorySegmentRequest.class, null);
679
680 if (body == null) {
681 return Response.error(400, "Invalid request body");
682 }
683
684 try {
685 UUID tmSegId = body.getTmSegId();
686 LayeredTextX source = body.getSource();
687 LayeredTextX target = body.getTarget();
688 UUID docId = body.getDocId();
689 String docFileName = body.getDocFileName();
690 UUID userId = body.getUserId();
691
692 if (!ControllerUtil.checkParam(tmSegId)) {
693 return Response.error(400, "Invalid request parameter TM Segment Id: " + tmSegId);
694 }
695
696 if (!ControllerUtil.checkParam(source)) {
697 return Response.error(400, "Invalid request, source is not specified");
698 }
699
700 if (!ControllerUtil.checkParam(target)) {
701 return Response.error(400, "Invalid request, target is not specified");
702 }
703
704 if (!ControllerUtil.checkParam(userId)) {
705 return Response.error(400, "Invalid request parameter User Id: " + userId);
706 }
707
708 SolrDocument tmDoc = SolrUtil.getTmByTmId(tmId);
709
710 if (tmDoc == null) {
711 return Response.error(404, "TM not found, tmId: " + tmId);
712 }
713
714
715 String segmentSolrId = SolrUtil.buildTmSegSolrId(tmId, source.getTextWithCodes());
716
717
718 try {
719 SolrClient solrClient = AthIndex.getSolr().getClient();
720 SolrQuery checkQuery = new SolrQuery("id:\"" + segmentSolrId + "\"");
721 checkQuery.setRows(1);
722
723 QueryResponse checkResponse = solrClient.query(Const.SOLR_CORE_ATH_TM_SEGMENTS, checkQuery);
724 SolrDocumentList docs = checkResponse.getResults();
725
726 if (docs != null && !docs.isEmpty()) {
727
728 SolrDocument existingDoc = docs.get(0);
729
730 UUID existingTmSegId = AthUtil.safeToUuid(
731 SolrUtil.safeGetField(existingDoc, Const.ATH_PROP_TM_SEG_ID, null), null);
732
733 if (existingTmSegId != null) {
734 Log.info(getClass(),
735 "TM segment with source='{}' already exists (tmSegId={}), updating instead",
736 source.getTextWithCodes(), existingTmSegId);
737
738
739 UpdateTranslationMemorySegmentRequest updateBody = new UpdateTranslationMemorySegmentRequest();
740 updateBody.setTmSegId(existingTmSegId);
741 updateBody.setDocId(docId);
742 updateBody.setDocFileName(docFileName);
743 updateBody.setTarget(target);
744 updateBody.setUserId(userId);
745
746 JsonNode updateBodyNode = JacksonUtil.makeNode(updateBody);
747
748
749 return updateTranslationMemorySegment(request, tmId, existingTmSegId, updateBodyNode);
750 }
751 }
752 } catch (Exception e) {
753 Log.warn(getClass(), "Error checking for existing TM segment: {}", e.getMessage());
754
755 }
756
757
758 SolrClient solrClient = AthIndex.getSolr().getClient();
759
760 String srcLang = SolrUtil.safeGetField(tmDoc, Const.ATH_PROP_SRC_LANG, null);
761 String trgLang = SolrUtil.safeGetField(tmDoc, Const.ATH_PROP_TRG_LANG, null);
762 String tmFileName = SolrUtil.safeGetField(tmDoc, Const.ATH_PROP_TM_FILE_NAME, null);
763
764 TmSegment segment = new TmSegment();
765
766 segment.setId(segmentSolrId);
767 segment.setTmSegId(tmSegId);
768 segment.setTmId(tmId);
769 segment.setTmFileName(tmFileName);
770
771 if (docId != null) {
772 segment.setDocId(docId);
773 }
774
775 if (docFileName != null) {
776 segment.setDocFileName(docFileName);
777 }
778
779 segment.setSrcLang(srcLang);
780 segment.setTrgLang(trgLang);
781 segment.setSource(source);
782 segment.setTarget(target);
783 segment.setCreatedBy(userId);
784 segment.setCreatedAt(new Date());
785
786 SolrInputDocument solrDoc = toSolrDoc(segment);
787 solrClient.add(Const.SOLR_CORE_ATH_TM_SEGMENTS, solrDoc);
788 solrClient.commit(Const.SOLR_CORE_ATH_TM_SEGMENTS);
789
790
791 SolrInputDocument updateDoc = SolrUtil.toInputDocument(SolrUtil.getTmByTmId(tmId));
792 updateDoc.setField(Const.ATH_PROP_UPDATED_BY, userId.toString());
793 updateDoc.setField(Const.ATH_PROP_UPDATED_AT, new Date());
794
795 solrClient.add(Const.SOLR_CORE_ATH_TMS, updateDoc);
796 solrClient.commit(Const.SOLR_CORE_ATH_TMS);
797
798 return Response.success(201, "TM segment created successfully");
799
800 } catch (Exception e) {
801 return Response.error(500, "Error creating TM segment -- " + e.getMessage());
802 }
803 }
804
805 public ResponseContext getTranslationMemorySegment(RequestContext request, UUID tmId,
806 UUID tmSegId) {
807
808 if (!ControllerUtil.checkParam(tmId)) {
809 return Response.error(400, "Invalid request parameter TM Id: " + tmId);
810 }
811
812 if (!ControllerUtil.checkParam(tmSegId)) {
813 return Response.error(400, "Invalid request parameter Segment Id: " + tmSegId);
814 }
815
816 String query = Log.format("tmId:\"{}\" AND tmSegId:\"{}\"", tmId, tmSegId);
817
818 try {
819 QueryResponse response = AthIndex.getMany(Const.SOLR_CORE_ATH_TM_SEGMENTS, query, null,
820 QueryResponse.class);
821
822 SolrDocumentList docList = response.getResults();
823
824 if (docList.isEmpty()) {
825 return Response.error(404, "TM segment not found");
826 }
827
828 SolrDocument doc = docList.get(0);
829 TmSegment segment = toTmSegment(doc);
830
831 if (segment == null) {
832 return Response.error(500, "Error parsing segment data");
833 }
834
835 TmSegmentWrapper wrapper = new TmSegmentWrapper();
836 wrapper.setTmSegment(segment);
837
838 return Response.success(200, wrapper);
839
840 } catch (Exception e) {
841 return Response.error(500, "Error fetching TM segment -- " + e.getMessage());
842 }
843 }
844
845 public ResponseContext updateTranslationMemorySegment(RequestContext request, UUID tmId,
846 UUID tmSegId, JsonNode bodyNode) {
847
848 if (!ControllerUtil.checkParam(tmId)) {
849 return Response.error(400, "Invalid request parameter TM Id: " + tmId);
850 }
851
852 if (!ControllerUtil.checkParam(tmSegId)) {
853 return Response.error(400, "Invalid request parameter Segment Id: " + tmSegId);
854 }
855
856 if (!ControllerUtil.checkParam(bodyNode)) {
857 return Response.error(400, "Invalid request body");
858 }
859
860 UpdateTranslationMemorySegmentRequest body = AthUtil.safeFromJsonNode(bodyNode,
861 UpdateTranslationMemorySegmentRequest.class, null);
862
863 if (body == null) {
864 return Response.error(400, "Invalid request body");
865 }
866
867 try {
868 UUID tmSegIdFromBody = body.getTmSegId();
869 UUID docId = body.getDocId();
870 String docFileName = body.getDocFileName();
871 LayeredTextX target = body.getTarget();
872 UUID userId = body.getUserId();
873
874 if (!ControllerUtil.checkParam(target)) {
875 return Response.error(400, "Invalid request, target is not specified");
876 }
877
878 if (!ControllerUtil.checkParam(userId)) {
879 return Response.error(400, "Invalid request parameter User Id: " + userId);
880 }
881
882 if (tmSegIdFromBody != null && !tmSegIdFromBody.equals(tmSegId)) {
883 return Response.error(400, "TM Segment Id mismatch");
884 }
885
886 if (docId != null) {
887 String docQuery = Log.format("docId:\"{}\"", docId);
888 if (SolrUtil.getNumDocuments(Const.SOLR_CORE_ATH_TMS, docQuery) <= 0) {
889 return Response.error(404, "Document not found, docId: " + docId);
890 }
891 }
892
893 String query = Log.format("tmId:\"{}\" AND tmSegId:\"{}\"", tmId, tmSegId);
894
895 QueryResponse response = AthIndex.getMany(Const.SOLR_CORE_ATH_TM_SEGMENTS, query, null,
896 QueryResponse.class);
897
898 SolrDocumentList docList = response.getResults();
899
900 if (docList.isEmpty()) {
901 return Response.error(404, "TM segment not found");
902 }
903
904 SolrDocument originalDoc = docList.get(0);
905 TmSegment segment = toTmSegment(originalDoc);
906
907 segment.setTarget(target);
908 if (docId != null) {
909 segment.setDocId(docId);
910 }
911 if (docFileName != null) {
912 segment.setDocFileName(docFileName);
913 }
914 segment.setUpdatedBy(userId);
915 segment.setUpdatedAt(new Date());
916
917 SolrInputDocument solrDoc = toSolrDoc(segment);
918 SolrClient solrClient = AthIndex.getSolr().getClient();
919 solrClient.add(Const.SOLR_CORE_ATH_TM_SEGMENTS, solrDoc);
920 solrClient.commit(Const.SOLR_CORE_ATH_TM_SEGMENTS);
921
922
923 SolrInputDocument updateDoc = SolrUtil.toInputDocument(SolrUtil.getTmByTmId(tmId));
924
925 updateDoc.setField(Const.ATH_PROP_UPDATED_BY, userId.toString());
926 updateDoc.setField(Const.ATH_PROP_UPDATED_AT, new Date());
927
928 solrClient.add(Const.SOLR_CORE_ATH_TMS, updateDoc);
929 solrClient.commit(Const.SOLR_CORE_ATH_TMS);
930
931 return Response.success(200, "TM segment updated successfully");
932
933 } catch (Exception e) {
934 return Response.error(500, "Error updating TM segment -- " + e.getMessage());
935 }
936 }
937
938 public ResponseContext deleteTranslationMemorySegment(RequestContext request, UUID tmId,
939 UUID tmSegId) {
940
941 if (!ControllerUtil.checkParam(tmId)) {
942 return Response.error(400, "Invalid request parameter TM Id: " + tmId);
943 }
944
945 if (!ControllerUtil.checkParam(tmSegId)) {
946 return Response.error(400, "Invalid request parameter Segment Id: " + tmSegId);
947 }
948
949 String query = Log.format("tmId:\"{}\" AND tmSegId:\"{}\"", tmId, tmSegId);
950
951 try {
952 if (SolrUtil.getNumDocuments(Const.SOLR_CORE_ATH_TM_SEGMENTS, query) <= 0) {
953 return Response.error(404, "TM segment not found");
954 }
955
956 SolrClient solrClient = AthIndex.getSolr().getClient();
957 AthIndex.deleteByQuery(Const.SOLR_CORE_ATH_TM_SEGMENTS, query);
958 solrClient.commit(Const.SOLR_CORE_ATH_TM_SEGMENTS);
959
960
961 SolrInputDocument updateDoc = SolrUtil.toInputDocument(SolrUtil.getTmByTmId(tmId));
962
963 updateDoc.setField(Const.ATH_PROP_UPDATED_AT, new Date());
964
965 solrClient.add(Const.SOLR_CORE_ATH_TMS, updateDoc);
966 solrClient.commit(Const.SOLR_CORE_ATH_TMS);
967
968 return Response.success(204, "TM segment deleted successfully");
969
970 } catch (Exception e) {
971 String st = Log.format("Error deleting TM segment (tmId={}, segId={}) -- {}",
972 tmId, tmSegId, e.getMessage());
973
974 Log.error(this.getClass(), e, st);
975 return Response.error(500, st);
976 }
977 }
978
979
980
981 private TmSegment toTmSegment(SolrDocument doc) {
982 try {
983 TmSegment segment = new TmSegment();
984
985
986 segment.setId(SolrUtil.safeGetField(doc, Const.ATH_PROP_SOLR_ID, null));
987
988 segment.setTmSegId(AthUtil.safeToUuid(
989 SolrUtil.safeGetField(doc, Const.ATH_PROP_TM_SEG_ID, null), null));
990
991 segment.setTmId(AthUtil.safeToUuid(
992 SolrUtil.safeGetField(doc, Const.ATH_PROP_TM_ID, null), null));
993
994 segment.setTmFileName(SolrUtil.safeGetField(doc, Const.ATH_PROP_TM_FILE_NAME, null));
995
996 segment.setDocId(AthUtil.safeToUuid(
997 SolrUtil.safeGetField(doc, Const.ATH_PROP_DOC_ID, null), null));
998
999 segment.setDocFileName(SolrUtil.safeGetField(doc, Const.ATH_PROP_DOC_FILE_NAME, null));
1000
1001 segment.setSrcLang(SolrUtil.safeGetField(doc, Const.ATH_PROP_SRC_LANG, null));
1002
1003 segment.setTrgLang(SolrUtil.safeGetField(doc, Const.ATH_PROP_TRG_LANG, null));
1004
1005
1006 String sourceJson = doc.getFieldValue(Const.ATH_PROP_SOURCE_JSON) != null
1007 ? doc.getFieldValue(Const.ATH_PROP_SOURCE_JSON).toString()
1008 : null;
1009
1010 String targetJson = doc.getFieldValue(Const.ATH_PROP_TARGET_JSON) != null
1011 ? doc.getFieldValue(Const.ATH_PROP_TARGET_JSON).toString()
1012 : null;
1013
1014 if (sourceJson != null) {
1015 LayeredTextX slt = JacksonUtil.fromJson(sourceJson, LayeredTextX.class);
1016 segment.setSource(slt);
1017 }
1018
1019 if (targetJson != null) {
1020 LayeredTextX tlt = JacksonUtil.fromJson(targetJson, LayeredTextX.class);
1021 segment.setTarget(tlt);
1022 }
1023
1024
1025 segment.setCreatedAt(AthUtil.safeToDate(doc.get(Const.ATH_PROP_CREATED_AT), null));
1026 segment.setUpdatedAt(AthUtil.safeToDate(doc.get(Const.ATH_PROP_UPDATED_AT), null));
1027
1028 segment.setCreatedBy(AthUtil.safeToUuid(
1029 SolrUtil.safeGetField(doc, Const.ATH_PROP_CREATED_BY, null), null));
1030
1031 segment.setUpdatedBy(AthUtil.safeToUuid(
1032 SolrUtil.safeGetField(doc, Const.ATH_PROP_UPDATED_BY, null), null));
1033
1034 return segment;
1035
1036 } catch (Exception e) {
1037 Log.error(this.getClass(), e, "Error converting Solr document to TmSegment");
1038 return null;
1039 }
1040 }
1041
1042 private SolrInputDocument toSolrDoc(TmSegment segment) throws AthException {
1043 SolrInputDocument doc = new SolrInputDocument();
1044
1045 if (segment.getId() != null) {
1046 doc.addField(Const.ATH_PROP_SOLR_ID, segment.getId());
1047 }
1048
1049 if (segment.getTmSegId() != null) {
1050 doc.addField(Const.ATH_PROP_TM_SEG_ID, segment.getTmSegId().toString());
1051 }
1052
1053 if (segment.getTmId() != null) {
1054 doc.addField(Const.ATH_PROP_TM_ID, segment.getTmId().toString());
1055 }
1056
1057 if (segment.getTmFileName() != null) {
1058 doc.addField(Const.ATH_PROP_TM_FILE_NAME, segment.getTmFileName());
1059 }
1060
1061 if (segment.getDocId() != null) {
1062 doc.addField(Const.ATH_PROP_DOC_ID, segment.getDocId().toString());
1063 }
1064
1065 if (segment.getDocFileName() != null) {
1066 doc.addField(Const.ATH_PROP_DOC_FILE_NAME, segment.getDocFileName());
1067 }
1068
1069 if (segment.getSrcLang() != null) {
1070 doc.addField(Const.ATH_PROP_SRC_LANG, segment.getSrcLang());
1071 }
1072
1073 if (segment.getTrgLang() != null) {
1074 doc.addField(Const.ATH_PROP_TRG_LANG, segment.getTrgLang());
1075 }
1076
1077 ITextUnit tu = new TextUnit(segment.getId() != null ? segment.getId() : "temp");
1078
1079 if (segment.getSource() != null) {
1080 String sourceJson = JacksonUtil.toJson(segment.getSource(), false);
1081 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_SOURCE_JSON, sourceJson);
1082
1083 if (segment.getSource().getText() != null) {
1084 doc.addField(Const.ATH_PROP_SOURCE, segment.getSource().getText());
1085 }
1086
1087 if (segment.getSource().getTextWithCodes() != null) {
1088 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_SOURCE_WITH_CODES,
1089 segment.getSource().getTextWithCodes());
1090 }
1091 }
1092
1093 if (segment.getTarget() != null) {
1094 String targetJson = JacksonUtil.toJson(segment.getTarget(), false);
1095 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_TARGET_JSON, targetJson);
1096
1097 if (segment.getTarget().getText() != null) {
1098 doc.addField(Const.ATH_PROP_TARGET, segment.getTarget().getText());
1099 }
1100
1101 if (segment.getTarget().getTextWithCodes() != null) {
1102 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_TARGET_WITH_CODES,
1103 segment.getTarget().getTextWithCodes());
1104 }
1105 }
1106
1107 if (segment.getCreatedAt() != null) {
1108 doc.addField(Const.ATH_PROP_CREATED_AT, segment.getCreatedAt());
1109 }
1110
1111 if (segment.getUpdatedAt() != null) {
1112 doc.addField(Const.ATH_PROP_UPDATED_AT, segment.getCreatedAt());
1113 }
1114
1115 if (segment.getCreatedBy() != null) {
1116 doc.addField(Const.ATH_PROP_CREATED_BY, segment.getCreatedBy().toString());
1117 }
1118
1119 if (segment.getUpdatedBy() != null) {
1120 doc.addField(Const.ATH_PROP_UPDATED_BY, segment.getUpdatedBy().toString());
1121 }
1122
1123 return doc;
1124 }
1125 }