1 package com.acumenvelocity.ath.solr.doc;
2
3 import java.util.Date;
4 import java.util.List;
5 import java.util.UUID;
6
7 import org.apache.solr.client.solrj.SolrClient;
8 import org.apache.solr.common.SolrDocument;
9 import org.apache.solr.common.SolrInputDocument;
10
11 import com.acumenvelocity.ath.common.Const;
12 import com.acumenvelocity.ath.common.ConversionUtil;
13 import com.acumenvelocity.ath.common.JacksonUtil;
14 import com.acumenvelocity.ath.common.Log;
15 import com.acumenvelocity.ath.common.SolrUtil;
16 import com.acumenvelocity.ath.model.MtTargetInfo;
17 import com.acumenvelocity.ath.model.Origin;
18 import com.acumenvelocity.ath.model.x.LayeredTextX;
19 import com.acumenvelocity.ath.solr.AthIndex;
20 import com.acumenvelocity.ath.solr.SolrIndexWriterStep;
21
22 import net.sf.okapi.common.Event;
23 import net.sf.okapi.common.LocaleId;
24 import net.sf.okapi.common.annotation.AltTranslation;
25 import net.sf.okapi.common.annotation.AltTranslationsAnnotation;
26 import net.sf.okapi.common.query.MatchType;
27 import net.sf.okapi.common.resource.ITextUnit;
28 import net.sf.okapi.common.resource.Segment;
29
30 public class SolrDocWriterStep extends SolrIndexWriterStep {
31
32 private UUID docId;
33 private String docFileName;
34 private UUID userId;
35 private boolean newDoc;
36 private long position;
37
38 public SolrDocWriterStep(UUID docId, String docFileName, UUID userId, boolean newDoc) {
39 super(AthIndex.getIndex().getClient(), Const.ATH_SOLR_BATCH_SIZE,
40 Const.SOLR_CORE_ATH_DOC_SEGMENTS, true, false);
41
42 this.docId = docId;
43 this.docFileName = docFileName;
44 this.userId = userId;
45 this.newDoc = newDoc;
46 }
47
48 @Override
49 public String getName() {
50 return "Solr Document Writer Step";
51 }
52
53 @Override
54 public String getDescription() {
55 return "Writes text units to Solr in batches";
56 }
57
58 @Override
59 protected Event handleStartDocument(Event event) {
60
61 position = 1;
62 return super.handleStartDocument(event);
63 }
64
65 @Override
66 protected Event handleEndBatch(Event event) {
67 Event ev = super.handleEndBatch(event);
68
69
70
71 SolrDocument existingDoc = SolrUtil.getDocumentByDocId(docId);
72 SolrInputDocument doc = SolrUtil.toInputDocument(existingDoc);
73 doc.setField(Const.ATH_PROP_UPDATED_AT, new Date());
74
75 try {
76 AthIndex.getSolr().getClient().add(Const.SOLR_CORE_ATH_DOCS, doc);
77 AthIndex.getSolr().getClient().commit(Const.SOLR_CORE_ATH_DOCS);
78
79 } catch (Exception e) {
80 Log.warn(SolrIndexWriterStep.class, "Unable to update document {} -- {}", docId,
81 e.getMessage());
82 }
83
84 return ev;
85 }
86
87 @Override
88 protected SolrInputDocument getSolrDocument(SolrClient solrClient, String coreName,
89 String sourceWithCodes) {
90
91 SolrInputDocument doc = null;
92
93 if (newDoc) {
94 doc = new SolrInputDocument();
95
96 } else {
97 try {
98 doc = SolrUtil.getDocumentBySolrId(solrClient, coreName,
99 SolrUtil.buildDocSegSolrId(docId, position));
100
101 } catch (Exception e) {
102 Log.warn(getClass(), "Error getting a Solr document for '{}' (position: {}) -- {}",
103 sourceWithCodes, position, e.getMessage());
104 }
105
106 if (doc == null) {
107 doc = new SolrInputDocument();
108 }
109 }
110
111 return doc;
112 }
113
114 @Override
115 protected void populateSolrDocument(SolrInputDocument doc, ITextUnit tu, Segment sseg,
116 Segment tseg, LocaleId srcLoc, LocaleId trgLoc) throws Exception {
117
118
119 if (sseg == null) {
120 return;
121 }
122
123 if (tseg == null && isRequireTarget()) {
124 position++;
125 return;
126 }
127
128
129 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_SOLR_ID,
130 SolrUtil.buildDocSegSolrId(docId, position));
131
132 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_DOC_SEG_ID, UUID.randomUUID().toString());
133
134 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_DOC_ID, docId.toString());
135 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_TU_ID, tu.getId());
136 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_DOC_FILE_NAME, docFileName);
137
138 doc.setField(Const.ATH_PROP_POSITION, position);
139
140 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_SRC_LANG, srcLoc.toString());
141 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_TRG_LANG, trgLoc.toString());
142
143 LayeredTextX slt = ConversionUtil.toLayeredText(sseg.getContent(), srcLoc);
144 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_SOURCE, slt.getText());
145 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_SOURCE_WITH_CODES, slt.getTextWithCodes());
146 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_SOURCE_JSON, JacksonUtil.toJson(slt, false));
147
148 if (tseg != null) {
149 LayeredTextX tlt = ConversionUtil.toLayeredText(tseg.getContent(), trgLoc);
150 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_TARGET, tlt.getText(), false);
151
152 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_TARGET_WITH_CODES, tlt.getTextWithCodes(),
153 false);
154
155 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_TARGET_JSON, JacksonUtil.toJson(tlt, false));
156
157 AltTranslationsAnnotation ata = tseg.getAnnotation(AltTranslationsAnnotation.class);
158
159
160 if (ata != null) {
161 ata.sort();
162 AltTranslation at = ata.getFirst();
163
164 if (at.getType() == Const.AL_MATCH_TYPE) {
165 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_ORIGIN, Origin.AL.toString());
166
167 } else if (at.getType() == MatchType.MT) {
168 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_ORIGIN, Origin.MT.toString());
169
170 } else if (at.getCombinedScore() > 0) {
171 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_ORIGIN, Origin.TM.toString());
172 doc.setField(Const.ATH_PROP_TM_MATCH_SCORE, at.getCombinedScore());
173 }
174
175 List<MtTargetInfo> altTrans = ConversionUtil.toMtTargets(ata);
176
177 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_ALT_TRANS_JSON,
178 JacksonUtil.toJson(altTrans, false));
179
180 doc.setField(Const.ATH_PROP_ALT_TRANS_INDEX, -1);
181 }
182 }
183
184 if (newDoc) {
185 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_CREATED_BY, userId.toString());
186 doc.setField(Const.ATH_PROP_CREATED_AT, new Date());
187
188 } else {
189 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_UPDATED_BY, userId.toString());
190 doc.setField(Const.ATH_PROP_UPDATED_AT, new Date());
191 }
192
193 position++;
194 }
195
196 }