View Javadoc
1   package com.acumenvelocity.ath.solr.doc;
2   
3   import java.util.Date;
4   import java.util.List;
5   import java.util.UUID;
6   
7   import org.apache.solr.client.solrj.SolrClient;
8   import org.apache.solr.common.SolrDocument;
9   import org.apache.solr.common.SolrInputDocument;
10  
11  import com.acumenvelocity.ath.common.Const;
12  import com.acumenvelocity.ath.common.ConversionUtil;
13  import com.acumenvelocity.ath.common.JacksonUtil;
14  import com.acumenvelocity.ath.common.Log;
15  import com.acumenvelocity.ath.common.SolrUtil;
16  import com.acumenvelocity.ath.model.MtTargetInfo;
17  import com.acumenvelocity.ath.model.Origin;
18  import com.acumenvelocity.ath.model.x.LayeredTextX;
19  import com.acumenvelocity.ath.solr.AthIndex;
20  import com.acumenvelocity.ath.solr.SolrIndexWriterStep;
21  
22  import net.sf.okapi.common.Event;
23  import net.sf.okapi.common.LocaleId;
24  import net.sf.okapi.common.annotation.AltTranslation;
25  import net.sf.okapi.common.annotation.AltTranslationsAnnotation;
26  import net.sf.okapi.common.query.MatchType;
27  import net.sf.okapi.common.resource.ITextUnit;
28  import net.sf.okapi.common.resource.Segment;
29  
30  public class SolrDocWriterStep extends SolrIndexWriterStep {
31  
32    private UUID docId;
33    private String docFileName;
34    private UUID userId;
35    private boolean newDoc;
36    private long position;
37  
38    public SolrDocWriterStep(UUID docId, String docFileName, UUID userId, boolean newDoc) {
39      super(AthIndex.getIndex().getClient(), Const.ATH_SOLR_BATCH_SIZE,
40          Const.SOLR_CORE_ATH_DOC_SEGMENTS, true, false);
41  
42      this.docId = docId;
43      this.docFileName = docFileName;
44      this.userId = userId;
45      this.newDoc = newDoc;
46    }
47  
48    @Override
49    public String getName() {
50      return "Solr Document Writer Step";
51    }
52  
53    @Override
54    public String getDescription() {
55      return "Writes text units to Solr in batches";
56    }
57  
58    @Override
59    protected Event handleStartDocument(Event event) {
60      // Reset the position within the document
61      position = 1;
62      return super.handleStartDocument(event);
63    }
64  
65    @Override
66    protected Event handleEndBatch(Event event) {
67      Event ev = super.handleEndBatch(event);
68  
69      // Update document's updated_at to current time so the modified segments summary returns an
70      // empty list (the newly imported document has unmodified segments only)
71      SolrDocument existingDoc = SolrUtil.getDocumentByDocId(docId);
72      SolrInputDocument doc = SolrUtil.toInputDocument(existingDoc);
73      doc.setField(Const.ATH_PROP_UPDATED_AT, new Date());
74  
75      try {
76        AthIndex.getSolr().getClient().add(Const.SOLR_CORE_ATH_DOCS, doc);
77        AthIndex.getSolr().getClient().commit(Const.SOLR_CORE_ATH_DOCS);
78  
79      } catch (Exception e) {
80        Log.warn(SolrIndexWriterStep.class, "Unable to update document {} -- {}", docId,
81            e.getMessage());
82      }
83  
84      return ev;
85    }
86  
87    @Override
88    protected SolrInputDocument getSolrDocument(SolrClient solrClient, String coreName,
89        String sourceWithCodes) {
90  
91      SolrInputDocument doc = null;
92  
93      if (newDoc) {
94        doc = new SolrInputDocument();
95  
96      } else {
97        try {
98          doc = SolrUtil.getDocumentBySolrId(solrClient, coreName,
99              SolrUtil.buildDocSegSolrId(docId, position));
100 
101       } catch (Exception e) {
102         Log.warn(getClass(), "Error getting a Solr document for '{}' (position: {}) -- {}",
103             sourceWithCodes, position, e.getMessage());
104       }
105 
106       if (doc == null) {
107         doc = new SolrInputDocument();
108       }
109     }
110 
111     return doc;
112   }
113 
114   @Override
115   protected void populateSolrDocument(SolrInputDocument doc, ITextUnit tu, Segment sseg,
116       Segment tseg, LocaleId srcLoc, LocaleId trgLoc) throws Exception {
117 
118     // XXX Keep in sync with BaseMergerStep
119     if (sseg == null) {
120       return;
121     }
122 
123     if (tseg == null && isRequireTarget()) {
124       position++;
125       return;
126     }
127 
128     // Unique id (docId + position) to overwrite an existing doc segment if the position is the same
129     SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_SOLR_ID,
130         SolrUtil.buildDocSegSolrId(docId, position));
131 
132     SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_DOC_SEG_ID, UUID.randomUUID().toString());
133 
134     SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_DOC_ID, docId.toString());
135     SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_TU_ID, tu.getId());
136     SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_DOC_FILE_NAME, docFileName);
137 
138     doc.setField(Const.ATH_PROP_POSITION, position);
139 
140     SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_SRC_LANG, srcLoc.toString());
141     SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_TRG_LANG, trgLoc.toString());
142 
143     LayeredTextX slt = ConversionUtil.toLayeredText(sseg.getContent(), srcLoc);
144     SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_SOURCE, slt.getText());
145     SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_SOURCE_WITH_CODES, slt.getTextWithCodes());
146     SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_SOURCE_JSON, JacksonUtil.toJson(slt, false));
147 
148     if (tseg != null) {
149       LayeredTextX tlt = ConversionUtil.toLayeredText(tseg.getContent(), trgLoc);
150       SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_TARGET, tlt.getText(), false);
151 
152       SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_TARGET_WITH_CODES, tlt.getTextWithCodes(),
153           false);
154 
155       SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_TARGET_JSON, JacksonUtil.toJson(tlt, false));
156 
157       AltTranslationsAnnotation ata = tseg.getAnnotation(AltTranslationsAnnotation.class);
158 
159       // Write origin
160       if (ata != null) {
161         ata.sort();
162         AltTranslation at = ata.getFirst();
163 
164         if (at.getType() == Const.AL_MATCH_TYPE) {
165           SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_ORIGIN, Origin.AL.toString());
166 
167         } else if (at.getType() == MatchType.MT) {
168           SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_ORIGIN, Origin.MT.toString());
169 
170         } else if (at.getCombinedScore() > 0) {
171           SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_ORIGIN, Origin.TM.toString());
172           doc.setField(Const.ATH_PROP_TM_MATCH_SCORE, at.getCombinedScore());
173         }
174 
175         List<MtTargetInfo> altTrans = ConversionUtil.toMtTargets(ata);
176 
177         SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_ALT_TRANS_JSON,
178             JacksonUtil.toJson(altTrans, false));
179 
180         doc.setField(Const.ATH_PROP_ALT_TRANS_INDEX, -1);
181       }
182     }
183 
184     if (newDoc) {
185       SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_CREATED_BY, userId.toString());
186       doc.setField(Const.ATH_PROP_CREATED_AT, new Date());
187 
188     } else {
189       SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_UPDATED_BY, userId.toString());
190       doc.setField(Const.ATH_PROP_UPDATED_AT, new Date());
191     }
192 
193     position++;
194   }
195 
196 }