001    package org.cumulus4j.store.datastoreversion.command;
002    
003    import java.util.ArrayList;
004    import java.util.Comparator;
005    import java.util.HashMap;
006    import java.util.List;
007    import java.util.Map;
008    import java.util.Properties;
009    import java.util.Set;
010    
011    import javax.jdo.PersistenceManager;
012    import javax.jdo.Query;
013    
014    import org.cumulus4j.store.Cumulus4jPersistenceHandler;
015    import org.cumulus4j.store.Cumulus4jStoreManager;
016    import org.cumulus4j.store.EncryptionHandler;
017    import org.cumulus4j.store.IndexEntryAction;
018    import org.cumulus4j.store.ProgressInfo;
019    import org.cumulus4j.store.WorkInProgressException;
020    import org.cumulus4j.store.crypto.CryptoContext;
021    import org.cumulus4j.store.datastoreversion.AbstractDatastoreVersionCommand;
022    import org.cumulus4j.store.datastoreversion.CommandApplyParam;
023    import org.cumulus4j.store.model.ClassMeta;
024    import org.cumulus4j.store.model.DataEntry;
025    import org.cumulus4j.store.model.FieldMeta;
026    import org.cumulus4j.store.model.IndexEntry;
027    import org.cumulus4j.store.model.ObjectContainer;
028    import org.datanucleus.ClassLoaderResolver;
029    import org.datanucleus.metadata.AbstractMemberMetaData;
030    import org.datanucleus.store.ExecutionContext;
031    import org.slf4j.Logger;
032    import org.slf4j.LoggerFactory;
033    
034    /**
035     * Delete all {@link IndexEntry}s from the datastore and then iterate all
036     * {@link DataEntry}s and re-index them.
037     * <p>
038     * TODO This class currently does not yet ensure that different keys are used. Thus,
039     * very likely the entire index uses only one single key. This should be improved
040     * in a future version.
041     * @author Marco หงุ่ยตระกูล-Schulze - marco at nightlabs dot de
042     */
043    public class RecreateIndex extends AbstractDatastoreVersionCommand
044    {
045            private static final Logger logger = LoggerFactory.getLogger(RecreateIndex.class);
046    
047            @Override
048            public int getCommandVersion() {
049                    return 1;
050            }
051    
052            @Override
053            public boolean isFinal() {
054                    return false;
055            }
056    
057            @Override
058            public boolean isKeyStoreDependent() {
059                    return true;
060            }
061    
062            private CommandApplyParam commandApplyParam;
063            private Properties workInProgressStateProperties;
064            private CryptoContext cryptoContext;
065            private PersistenceManager pmIndex;
066            private PersistenceManager pmData;
067            private long keyStoreRefID;
068    
069            private Set<Class<? extends IndexEntry>> indexEntryClasses;
070    
071            @Override
072            public void apply(CommandApplyParam commandApplyParam) {
073                    // The index only exists in the index-datastore (not in the index-datastore), hence we return immediately, if the
074                    // current datastore is not the index-datastore.
075                    this.commandApplyParam = commandApplyParam;
076                    PersistenceManager pm = commandApplyParam.getPersistenceManager();
077                    cryptoContext = commandApplyParam.getCryptoContext();
078                    if (pm != cryptoContext.getPersistenceManagerForIndex())
079                            return;
080    
081                    keyStoreRefID = cryptoContext.getKeyStoreRefID();
082                    pmIndex = commandApplyParam.getCryptoContext().getPersistenceManagerForIndex();
083                    pmData = commandApplyParam.getCryptoContext().getPersistenceManagerForData();
084                    workInProgressStateProperties = commandApplyParam.getWorkInProgressStateProperties();
085    
086                    deleteIndex();
087                    createIndex();
088            }
089    
090            protected static final String PROPERTY_DELETE_COMPLETE = "delete.complete";
091            protected static final String PROPERTY_DELETE_FROM_INDEX_ENTRY_ID = "delete.fromIndexEntryID";
092            protected static final String PROPERTY_CREATE_FROM_DATA_ENTRY_ID = "create.fromDataEntryID";
093    
094            protected void deleteIndex() {
095                    logger.debug("deleteIndex: Entered.");
096                    if (Boolean.parseBoolean(workInProgressStateProperties.getProperty(PROPERTY_DELETE_COMPLETE))) {
097                            logger.debug("deleteIndex: PROPERTY_DELETE_COMPLETE == true => quit.");
098                            return;
099                    }
100    
101                    final long indexEntryBlockSize = 100;
102                    Long maxIndexEntryIDObj = getMaxIndexEntryID();
103                    if (maxIndexEntryIDObj == null) {
104                            logger.debug("deleteIndex: There are no IndexEntry instances in the database => quit.");
105                    }
106                    else {
107                            final long maxIndexEntryID = maxIndexEntryIDObj;
108                            String fromIndexEntryStr = workInProgressStateProperties.getProperty(PROPERTY_DELETE_FROM_INDEX_ENTRY_ID);
109                            long fromIndexEntryID;
110                            if (fromIndexEntryStr != null) {
111                                    logger.info("deleteIndex: previous incomplete run found: fromIndexEntryStr={}", fromIndexEntryStr);
112                                    fromIndexEntryID = Long.parseLong(fromIndexEntryStr);
113                            }
114                            else {
115                                    final long minIndexEntryID = getMinIndexEntryID();
116                                    logger.info("deleteIndex: first run: minIndexEntryID={} maxIndexEntryID={}", minIndexEntryID, maxIndexEntryID);
117                                    fromIndexEntryID = minIndexEntryID;
118                            }
119                            while (fromIndexEntryID <= maxIndexEntryID - indexEntryBlockSize) {
120                                    long toIndexEntryIDExcl = fromIndexEntryID + indexEntryBlockSize;
121                                    deleteIndexForRange(fromIndexEntryID, toIndexEntryIDExcl);
122                                    fromIndexEntryID = toIndexEntryIDExcl;
123                                    if (commandApplyParam.isDatastoreVersionCommandApplyWorkInProgressTimeoutExceeded()) {
124                                            workInProgressStateProperties.setProperty(PROPERTY_DELETE_FROM_INDEX_ENTRY_ID, Long.toString(fromIndexEntryID));
125                                            throw new WorkInProgressException(new ProgressInfo());
126                                    }
127                            }
128                            deleteIndexForRange(fromIndexEntryID, null);
129                    }
130    
131                    workInProgressStateProperties.setProperty(PROPERTY_DELETE_COMPLETE, Boolean.TRUE.toString());
132                    logger.debug("deleteIndex: Leaving.");
133            }
134    
135            protected void deleteIndexForRange(long fromIndexEntryIDIncl, Long toIndexEntryIDExcl) {
136                    logger.info("deleteIndexForRange: Entered. fromIndexEntryIDIncl={} toIndexEntryIDExcl={}", fromIndexEntryIDIncl, toIndexEntryIDExcl);
137                    List<IndexEntry> indexEntries = getIndexEntries(fromIndexEntryIDIncl, toIndexEntryIDExcl);
138                    pmIndex.deletePersistentAll(indexEntries);
139                    logger.info("deleteIndexForRange: Leaving. fromIndexEntryIDIncl={} toIndexEntryIDExcl={}", fromIndexEntryIDIncl, toIndexEntryIDExcl);
140            }
141    
142            protected List<IndexEntry> getIndexEntries(long fromIndexEntryIDIncl, Long toIndexEntryIDExcl) {
143                    List<IndexEntry> result = new ArrayList<IndexEntry>();
144                    for (Class<? extends IndexEntry> indexEntryClass : getIndexEntryClasses()) {
145                            result.addAll(getIndexEntries(indexEntryClass, fromIndexEntryIDIncl, toIndexEntryIDExcl));
146                    }
147                    return result;
148            }
149    
150            protected List<IndexEntry> getIndexEntries(Class<? extends IndexEntry> indexEntryClass, long fromIndexEntryIDIncl, Long toIndexEntryIDExcl) {
151                    Query q = pmIndex.newQuery(indexEntryClass);
152                    StringBuilder filter = new StringBuilder();
153                    Map<String, Object> params = new HashMap<String, Object>(2);
154    
155                    filter.append("this.keyStoreRefID == :keyStoreRefID");
156                    params.put("keyStoreRefID", keyStoreRefID);
157    
158                    if (fromIndexEntryIDIncl > 0) { // required for GAE, because it throws an exception when querying for ID >= 0, saying that ID == 0 is illegal.
159                            filter.append(" && this.indexEntryID >= :fromIndexEntryIDIncl");
160                            params.put("fromIndexEntryIDIncl", fromIndexEntryIDIncl);
161                    }
162    
163                    if (toIndexEntryIDExcl != null) {
164                            filter.append(" && this.indexEntryID < :toIndexEntryIDExcl");
165                            params.put("toIndexEntryIDExcl", toIndexEntryIDExcl);
166                    }
167                    q.setFilter(filter.toString());
168                    q.setOrdering("this.indexEntryID ASC");
169    
170                    @SuppressWarnings("unchecked")
171                    List<IndexEntry> result = new ArrayList<IndexEntry>((List<IndexEntry>) q.executeWithMap(params));
172                    q.closeAll();
173                    return result;
174            }
175    
176            protected void createIndex()
177            {
178                    final long dataEntryBlockSize = 100;
179                    long fromDataEntryID;
180                    String fromDataEntryIDStr = workInProgressStateProperties.getProperty(PROPERTY_CREATE_FROM_DATA_ENTRY_ID);
181                    final long maxDataEntryID = getMaxDataEntryID();
182                    if (fromDataEntryIDStr != null) {
183                            fromDataEntryID = Long.parseLong(fromDataEntryIDStr);
184                    }
185                    else {
186                            final long minDataEntryID = getMinDataEntryID();
187                            fromDataEntryID = minDataEntryID;
188                    }
189                    while (fromDataEntryID <= maxDataEntryID - dataEntryBlockSize) {
190                            long toDataEntryIDExcl = fromDataEntryID + dataEntryBlockSize;
191                            createIndexForRange(fromDataEntryID, toDataEntryIDExcl);
192                            fromDataEntryID = toDataEntryIDExcl;
193    
194                            if (commandApplyParam.isDatastoreVersionCommandApplyWorkInProgressTimeoutExceeded()) {
195                                    workInProgressStateProperties.setProperty(PROPERTY_CREATE_FROM_DATA_ENTRY_ID, Long.toString(fromDataEntryID));
196                                    throw new WorkInProgressException(new ProgressInfo());
197                            }
198                    }
199                    createIndexForRange(fromDataEntryID, null);
200            }
201    
202            protected void createIndexForRange(long fromDataEntryIDIncl, Long toDataEntryIDExcl) {
203                    ExecutionContext ec = cryptoContext.getExecutionContext();
204                    ClassLoaderResolver clr = ec.getClassLoaderResolver();
205                    Cumulus4jStoreManager storeManager = commandApplyParam.getStoreManager();
206                    EncryptionHandler encryptionHandler = storeManager.getEncryptionHandler();
207                    Cumulus4jPersistenceHandler persistenceHandler = storeManager.getPersistenceHandler();
208                    IndexEntryAction addIndexEntryAction = persistenceHandler.getAddIndexEntryAction();
209                    List<DataEntryWithClassName> l = getDataEntries(fromDataEntryIDIncl, toDataEntryIDExcl);
210                    for (DataEntryWithClassName dataEntryWithClassName : l) {
211                            long dataEntryID = dataEntryWithClassName.getDataEntry().getDataEntryID();
212                            Class<?> clazz = clr.classForName(dataEntryWithClassName.getClassName());
213                            ClassMeta classMeta = storeManager.getClassMeta(ec, clazz);
214                            ObjectContainer objectContainer = encryptionHandler.decryptDataEntry(cryptoContext, dataEntryWithClassName.getDataEntry());
215                            for (Map.Entry<Long, Object> me : objectContainer.getFieldID2value().entrySet()) {
216                                    long fieldID = me.getKey();
217                                    Object fieldValue = me.getValue();
218                                    FieldMeta fieldMeta = classMeta.getFieldMeta(fieldID);
219                                    AbstractMemberMetaData dnMemberMetaData = fieldMeta.getDataNucleusMemberMetaData(ec);
220                                    addIndexEntryAction.perform(cryptoContext, dataEntryID, fieldMeta, dnMemberMetaData, classMeta, fieldValue);
221                            }
222                    }
223            }
224    
225            protected static final class DataEntryWithClassName {
226                    private DataEntry dataEntry;
227                    private String className;
228    
229                    public DataEntryWithClassName(DataEntry dataEntry, String className) {
230                            if (dataEntry == null)
231                                    throw new IllegalArgumentException("dataEntry == null");
232                            if (className == null)
233                                    throw new IllegalArgumentException("className == null");
234                            this.dataEntry = dataEntry;
235                            this.className = className;
236                    }
237                    /**
238                     * Get the {@link DataEntry}.
239                     * @return the {@link DataEntry}. Never <code>null</code>.
240                     */
241                    public DataEntry getDataEntry() {
242                            return dataEntry;
243                    }
244                    /**
245                     * Get the fully qualified class name of the persistence-capable object represented by
246                     * {@link #dataEntry}.
247                     * @return the fully qualified class name. Never <code>null</code>.
248                     */
249                    public String getClassName() {
250                            return className;
251                    }
252            }
253    
254            protected List<DataEntryWithClassName> getDataEntries(long fromDataEntryIDIncl, Long toDataEntryIDExcl) {
255                    Query q = pmData.newQuery(DataEntry.class);
256                    q.declareVariables(ClassMeta.class.getName() + " classMeta");
257                    q.setResult("this, classMeta.packageName, classMeta.simpleClassName");
258                    StringBuilder filter = new StringBuilder();
259                    Map<String, Object> params = new HashMap<String, Object>(2);
260    
261                    filter.append("this.classMeta_classID == classMeta.classID");
262                    filter.append(" && this.keyStoreRefID == :keyStoreRefID");
263                    params.put("keyStoreRefID", keyStoreRefID);
264    
265                    if (fromDataEntryIDIncl > 0) { // required for GAE, because it throws an exception when querying for ID >= 0, saying that ID == 0 is illegal.
266                            filter.append(" && this.dataEntryID >= :fromDataEntryIDIncl");
267                            params.put("fromDataEntryIDIncl", fromDataEntryIDIncl);
268                    }
269    
270                    if (toDataEntryIDExcl != null) {
271                            filter.append(" && this.dataEntryID < :toDataEntryIDExcl");
272                            params.put("toDataEntryIDExcl", toDataEntryIDExcl);
273                    }
274                    q.setFilter(filter.toString());
275                    q.setOrdering("this.dataEntryID ASC");
276    
277                    @SuppressWarnings("unchecked")
278                    List<Object[]> l = (List<Object[]>) q.executeWithMap(params);
279                    List<DataEntryWithClassName> result = new ArrayList<DataEntryWithClassName>(l.size());
280                    for (Object[] row : l) {
281                            if (row.length != 3)
282                                    throw new IllegalStateException(String.format("row.length == %s != 3", row.length));
283    
284                            result.add(new DataEntryWithClassName(
285                                            (DataEntry)row[0],
286                                            ClassMeta.getClassName((String)row[1], (String)row[2])
287                            ));
288                    }
289                    q.closeAll();
290                    return result;
291            }
292    
293            protected long getMinDataEntryID() {
294                    Long result = getMinMaxDataEntryID("min");
295                    if (result == null)
296                            return 0;
297                    return result;
298            }
299    
300            protected long getMaxDataEntryID() {
301                    Long result = getMinMaxDataEntryID("max");
302                    if (result == null)
303                            return 0;
304                    return result;
305            }
306    
307            protected Long getMinMaxDataEntryID(String minMax) {
308                    Query q = pmData.newQuery(DataEntry.class);
309                    q.setResult(minMax + "(this.dataEntryID)");
310                    Long result = (Long) q.execute();
311                    return result;
312            }
313    
314            protected Long getMinIndexEntryID() {
315                    return getMinMaxIndexEntryID("min", new Comparator<Long>() {
316                            @Override
317                            public int compare(Long o1, Long o2) {
318                                    return o2.compareTo(o1);
319                            }
320                    });
321            }
322    
323            protected Long getMaxIndexEntryID() {
324                    return getMinMaxIndexEntryID("max", new Comparator<Long>() {
325                            @Override
326                            public int compare(Long o1, Long o2) {
327                                    return o1.compareTo(o2);
328                            }
329                    });
330            }
331    
332            protected Long getMinMaxIndexEntryID(String minMax, Comparator<Long> comparator) {
333                    Long result = null;
334                    for (Class<? extends IndexEntry> indexEntryClass : getIndexEntryClasses()) {
335                            Long minMaxIndexEntryID = getMinMaxIndexEntryID(indexEntryClass, minMax);
336                            if (minMaxIndexEntryID != null) {
337                                    if (result == null || comparator.compare(result, minMaxIndexEntryID) < 0)
338                                            result = minMaxIndexEntryID;
339                            }
340                    }
341                    return result;
342            }
343    
344            protected Long getMinMaxIndexEntryID(Class<? extends IndexEntry> indexEntryClass, String minMax) {
345                    Query q = pmIndex.newQuery(indexEntryClass);
346                    q.setResult(minMax + "(this.indexEntryID)");
347                    Long result = (Long) q.execute();
348                    return result;
349            }
350    
351            protected Set<Class<? extends IndexEntry>> getIndexEntryClasses() {
352                    if (indexEntryClasses == null)
353                            indexEntryClasses = ((Cumulus4jStoreManager)cryptoContext.getExecutionContext().getStoreManager()).getIndexFactoryRegistry().getIndexEntryClasses();
354    
355                    return indexEntryClasses;
356            }
357    }