001    /*
002     * Cumulus4j - Securing your data in the cloud - http://cumulus4j.org
003     * Copyright (C) 2011 NightLabs Consulting GmbH
004     *
005     * This program is free software: you can redistribute it and/or modify
006     * it under the terms of the GNU Affero General Public License as
007     * published by the Free Software Foundation, either version 3 of the
008     * License, or (at your option) any later version.
009     *
010     * This program is distributed in the hope that it will be useful,
011     * but WITHOUT ANY WARRANTY; without even the implied warranty of
012     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
013     * GNU Affero General Public License for more details.
014     *
015     * You should have received a copy of the GNU Affero General Public License
016     * along with this program.  If not, see <http://www.gnu.org/licenses/>.
017     */
018    package org.cumulus4j.store.query;
019    
020    import java.util.Collection;
021    import java.util.Deque;
022    import java.util.HashMap;
023    import java.util.HashSet;
024    import java.util.Iterator;
025    import java.util.LinkedList;
026    import java.util.List;
027    import java.util.Map;
028    import java.util.Set;
029    
030    import javax.jdo.PersistenceManager;
031    
032    import org.cumulus4j.store.Cumulus4jStoreManager;
033    import org.cumulus4j.store.EncryptionHandler;
034    import org.cumulus4j.store.PersistenceManagerConnection;
035    import org.cumulus4j.store.crypto.CryptoContext;
036    import org.cumulus4j.store.model.ClassMeta;
037    import org.cumulus4j.store.model.ClassMetaDAO;
038    import org.cumulus4j.store.model.DataEntry;
039    import org.cumulus4j.store.model.EmbeddedClassMeta;
040    import org.cumulus4j.store.query.eval.AbstractExpressionEvaluator;
041    import org.cumulus4j.store.query.eval.AndExpressionEvaluator;
042    import org.cumulus4j.store.query.eval.ComparisonExpressionEvaluator;
043    import org.cumulus4j.store.query.eval.InvokeExpressionEvaluator;
044    import org.cumulus4j.store.query.eval.LiteralEvaluator;
045    import org.cumulus4j.store.query.eval.NotExpressionEvaluator;
046    import org.cumulus4j.store.query.eval.OrExpressionEvaluator;
047    import org.cumulus4j.store.query.eval.ParameterExpressionEvaluator;
048    import org.cumulus4j.store.query.eval.PrimaryExpressionEvaluator;
049    import org.cumulus4j.store.query.eval.ResultDescriptor;
050    import org.cumulus4j.store.query.eval.SubqueryExpressionEvaluator;
051    import org.cumulus4j.store.query.eval.VariableExpressionEvaluator;
052    import org.datanucleus.ClassLoaderResolver;
053    import org.datanucleus.identity.IdentityUtils;
054    import org.datanucleus.metadata.AbstractClassMetaData;
055    import org.datanucleus.query.QueryUtils;
056    import org.datanucleus.query.compiler.QueryCompilation;
057    import org.datanucleus.query.expression.DyadicExpression;
058    import org.datanucleus.query.expression.Expression;
059    import org.datanucleus.query.expression.InvokeExpression;
060    import org.datanucleus.query.expression.Literal;
061    import org.datanucleus.query.expression.ParameterExpression;
062    import org.datanucleus.query.expression.PrimaryExpression;
063    import org.datanucleus.query.expression.SubqueryExpression;
064    import org.datanucleus.query.expression.VariableExpression;
065    import org.datanucleus.query.symbol.Symbol;
066    import org.datanucleus.store.ExecutionContext;
067    import org.datanucleus.store.query.Query;
068    
069    /**
070     * API-agnostic query implementation. An instance of this class performs the actual query.
071     * It is used by both APIs, JDO and JPA.
072     *
073     * @author Marco หงุ่ยตระกูล-Schulze - marco at nightlabs dot de
074     */
075    public abstract class QueryEvaluator
076    {
077            /** Name under which any set of results are stored in the state map. Used for aggregation. */
078            public static final String RESULTS_SET = "DATANUCLEUS_RESULTS_SET";
079    
080            private final String language;
081    
082            private String candidateAlias = "this";
083    
084            /** Underlying "string-based" query. */
085            private Query query;
086    
087            /** Compilation of the underlying query, that we are evaluating. */
088            private QueryCompilation compilation;
089    
090            /** Map of input parameter values, keyed by the parameter name. */
091            private Map<String, Object> parameterValues;
092    
093            /** Positional parameter that we are up to (-1 implies not being used). */
094            private Map<Integer, Symbol> paramSymbolByPosition = null;
095    
096            /** Map of state symbols for the query evaluation. */
097            private Map<String, Object> state;
098    
099            private ClassLoaderResolver clr;
100    
101            private ExecutionContext ec;
102    
103            private Cumulus4jStoreManager storeManager;
104    
105            private CryptoContext cryptoContext;
106    
107            private PersistenceManagerConnection pmConn;
108    
109            private EncryptionHandler encryptionHandler;
110    
111            private boolean complete = true;
112    
113            private Map<Symbol, EmbeddedClassMeta> symbol2ValueTypeEmbeddedClassMeta = null;
114    
115            /**
116             * @param language Query language (JDOQL, JPQL, etc)
117             * @param compilation generic compilation
118             * @param parameterValues Input values for the params
119             * @param clr ClassLoader resolver
120             * @param pmConn our <b>backend</b>-<code>PersistenceManager</code> connection(s).
121             * @param cryptoContext TODO
122             */
123            public QueryEvaluator(
124                            String language, Query query, QueryCompilation compilation, Map<String, Object> parameterValues,
125                            ClassLoaderResolver clr, PersistenceManagerConnection pmConn, CryptoContext cryptoContext)
126            {
127                    this.language = language;
128                    this.query = query;
129                    this.compilation = compilation;
130                    this.parameterValues = parameterValues;
131                    this.clr = clr;
132                    this.ec = query.getExecutionContext();
133                    this.storeManager = (Cumulus4jStoreManager) query.getStoreManager();
134                    this.pmConn = pmConn;
135                    this.cryptoContext = cryptoContext;
136                    this.encryptionHandler = storeManager.getEncryptionHandler();
137    
138                    this.candidateAlias = (compilation.getCandidateAlias() != null ? compilation.getCandidateAlias() : this.candidateAlias);
139    
140                    state = new HashMap<String, Object>();
141                    state.put(this.candidateAlias, query.getCandidateClass());
142    
143                    if (parameterValues != null && !parameterValues.isEmpty()) {
144                            Object paramKey = parameterValues.keySet().iterator().next();
145                            if (paramKey instanceof Integer) {
146                                    paramSymbolByPosition = new HashMap<Integer, Symbol>();
147                            }
148                    }
149            }
150    
151            public boolean isComplete() {
152                    return complete;
153            }
154    
155            public void setIncomplete() {
156                    this.complete = false;
157            }
158    
159            public String getLanguage() {
160                    return language;
161            }
162    
163            public String getCandidateAlias() {
164                    return candidateAlias;
165            }
166    
167            public Query getQuery() {
168                    return query;
169            }
170    
171            public QueryCompilation getCompilation() {
172                    return compilation;
173            }
174    
175            public Map<String, Object> getParameterValues() {
176                    return parameterValues;
177            }
178    
179            public Map<String, Object> getState() {
180                    return state;
181            }
182    
183            public ClassLoaderResolver getClassLoaderResolver() {
184                    return clr;
185            }
186    
187            public ExecutionContext getExecutionContext() {
188                    return ec;
189            }
190    
191            public Cumulus4jStoreManager getStoreManager() {
192                    return storeManager;
193            }
194    
195            public PersistenceManagerConnection getPersistenceManagerConnection() {
196                    return pmConn;
197            }
198    
199            public PersistenceManager getPersistenceManagerForData() {
200                    return pmConn.getDataPM();
201            }
202    
203            public PersistenceManager getPersistenceManagerForIndex() {
204                    return pmConn.getIndexPM();
205            }
206    
207            public EncryptionHandler getEncryptionHandler() {
208                    return encryptionHandler;
209            }
210    
211            private Deque<ResultDescriptor> resultDescriptors = new LinkedList<ResultDescriptor>();
212    
213            /**
214             * Push a {@link ResultDescriptor} onto the stack.
215             * @param resultDescriptor the descriptor to be pushed.
216             */
217            public void pushResultDescriptor(ResultDescriptor resultDescriptor)
218            {
219                    resultDescriptors.push(resultDescriptor);
220            }
221    
222            /**
223             * Pop a {@link ResultDescriptor} from the stack.
224             * @return the popped descriptor (which is the last one pushed).
225             */
226            public ResultDescriptor popResultDescriptor()
227            {
228                    return resultDescriptors.pop();
229            }
230    
231            public ClassMeta getValueTypeClassMeta(Symbol symbol, boolean throwExceptionIfNotFound) {
232                    ClassMeta classMeta = getValueTypeEmbeddedClassMeta(symbol);
233                    if (classMeta == null) {
234                            Class<?> clazz = getValueType(symbol, throwExceptionIfNotFound);
235                            classMeta = getStoreManager().getClassMeta(getExecutionContext(), clazz);
236                    }
237                    return classMeta;
238            }
239    
240            public EmbeddedClassMeta getValueTypeEmbeddedClassMeta(Symbol symbol) {
241                    if (symbol == null)
242                            throw new IllegalArgumentException("symbol == null");
243    
244                    if (symbol2ValueTypeEmbeddedClassMeta == null)
245                            return null;
246    
247                    return symbol2ValueTypeEmbeddedClassMeta.get(symbol);
248            }
249    
250            public void registerValueTypeEmbeddedClassMeta(Symbol symbol, EmbeddedClassMeta embeddedClassMeta) {
251                    if (symbol == null)
252                            throw new IllegalArgumentException("symbol == null");
253    
254                    if (embeddedClassMeta == null)
255                            return;
256    
257                    if (symbol2ValueTypeEmbeddedClassMeta == null)
258                            symbol2ValueTypeEmbeddedClassMeta = new HashMap<Symbol, EmbeddedClassMeta>();
259    
260                    symbol2ValueTypeEmbeddedClassMeta.put(symbol, embeddedClassMeta);
261            }
262    
263            /**
264             * Get a <code>Symbol</code>'s {@link Symbol#getValueType() valueType} by taking {@link ResultDescriptor}s into account.
265             * Delegates to {@link #getValueType(Symbol, boolean)} with <code>throwExceptionIfNotFound == true</code>.
266             *
267             * @param symbol the symbol whose {@link Symbol#getValueType() valueType} should be resolved.
268             * @return the type - never <code>null</code>.
269             * @see #getValueType(Symbol, boolean)
270             */
271            public Class<?> getValueType(Symbol symbol)
272            {
273                    return getValueType(symbol, true);
274            }
275    
276            /**
277             * <p>
278             * Get a <code>Symbol</code>'s {@link Symbol#getValueType() valueType} by taking {@link ResultDescriptor}s into account.
279             * </p>
280             * <p>
281             * This method (or alternatively {@link #getValueType(Symbol)}) should always be used instead of directly
282             * accessing {@link Symbol#getValueType()}!!! This allows for implicit variables (which are not declared).
283             * </p>
284             * <p>
285             * This method first checks, if {@link Symbol#getValueType()} returns a value and if so, returns it. Otherwise
286             * it searches the stack of {@link ResultDescriptor}s (maintained via {@link #pushResultDescriptor(ResultDescriptor)}
287             * and {@link #popResultDescriptor()}) and returns the first found {@link ResultDescriptor#getResultType()}.
288             * </p>
289             *
290             * @param symbol the symbol whose {@link Symbol#getValueType() valueType} should be resolved.
291             * @param throwExceptionIfNotFound whether to throw an {@link IllegalStateException} [exception type might be changed without notice!],
292             * if the type cannot be resolved. If <code>false</code> this method returns <code>null</code> instead.
293             * @return the type or <code>null</code>, if not resolvable and <code>throwExceptionIfNotFound == false</code>.
294             * @see #getValueType(Symbol)
295             * @see #pushResultDescriptor(ResultDescriptor)
296             * @see #popResultDescriptor()
297             */
298            public Class<?> getValueType(Symbol symbol, boolean throwExceptionIfNotFound)
299            {
300                    if (symbol.getValueType() != null)
301                            return symbol.getValueType();
302    
303                    for (ResultDescriptor resultDescriptor : resultDescriptors) {
304                            if (symbol.equals(resultDescriptor.getSymbol()))
305                                    return resultDescriptor.getResultType();
306                    }
307    
308                    if (symbol.getType() == Symbol.PARAMETER) {
309                            // Cater for implicit parameters where the generic compilation doesn't have the type
310                            if (paramSymbolByPosition != null) {
311                                    // Positional parameters
312                                    Iterator<Map.Entry<Integer, Symbol>> paramIter = paramSymbolByPosition.entrySet().iterator();
313                                    while (paramIter.hasNext()) {
314                                            Map.Entry<Integer, Symbol> entry = paramIter.next();
315                                            if (entry.getValue() == symbol) {
316                                                    return parameterValues.get(entry.getKey()).getClass();
317                                            }
318                                    }
319    
320                                    Integer nextPos = paramSymbolByPosition.size();
321                                    Object value = parameterValues.get(nextPos);
322                                    paramSymbolByPosition.put(nextPos, symbol);
323                                    return value.getClass();
324                            }
325                            else {
326                                    if (parameterValues.containsKey(symbol.getQualifiedName())) {
327                                            return parameterValues.get(symbol.getQualifiedName()).getClass();
328                                    }
329                            }
330                    }
331                    if (throwExceptionIfNotFound)
332                            throw new IllegalStateException("Could not determine the resultType of symbol \"" + symbol + "\"! If this is a variable, you might want to declare it.");
333    
334                    return null;
335            }
336    
337            protected abstract Collection<Object> evaluateSubquery(
338                            Query subquery, QueryCompilation compilation, Object outerCandidate
339            );
340    
341            public List<Object> execute()
342            {
343                    Class<?> candidateClass = query.getCandidateClass();
344                    boolean withSubclasses = query.isSubclasses();
345                    Set<ClassMeta> candidateClassMetas = QueryHelper.getCandidateClassMetas(storeManager, ec, candidateClass, withSubclasses);
346    
347                    // TODO I copied this from the JavaQueryEvaluator, but I'm not sure, whether we need this. Need to talk with Andy. Marco.
348                    // ...or analyse it ourselves (step through)...
349                    String[] subqueryAliases = compilation.getSubqueryAliases();
350                    if (subqueryAliases != null) {
351                            for (int i=0; i<subqueryAliases.length; ++i) {
352                                    // Evaluate subquery first
353                                    Query subquery = query.getSubqueryForVariable(subqueryAliases[i]).getQuery();
354                                    QueryCompilation subqueryCompilation = compilation.getCompilationForSubquery(subqueryAliases[i]);
355    
356                                    Collection<Object> subqueryResult = evaluateSubquery(subquery, subqueryCompilation, null);
357    
358                                    if (QueryUtils.queryReturnsSingleRow(subquery)) {
359                                            Iterator<Object> subqueryIterator = subqueryResult.iterator();
360                                            if (!subqueryIterator.hasNext()) // TODO simply use null in this case?!????
361                                                    throw new IllegalStateException("Subquery is expected to return a single row, but it returned an empty collection!");
362    
363                                            state.put(subqueryAliases[i], subqueryIterator.next());
364    
365                                            if (subqueryIterator.hasNext())
366                                                    throw new IllegalStateException("Subquery is expected to return only a single row, but it returned more than one!");
367                                    }
368                                    else
369                                            state.put(subqueryAliases[i], subqueryResult);
370                            }
371                    }
372    
373                    if (compilation.getExprFilter() == null) {
374                            // No filter - we want all that match the candidate classes.
375                            return QueryHelper.getAllPersistentObjectsForCandidateClasses(cryptoContext, getPersistenceManagerForData(), candidateClassMetas);
376                    }
377                    else {
378                            expressionEvaluator = createExpressionEvaluatorTree(compilation.getExprFilter());
379                            Symbol resultSymbol = getCompilation().getSymbolTable().getSymbol(getCandidateAlias());
380                            if (resultSymbol == null)
381                                    throw new IllegalStateException("getCompilation().getSymbolTable().getSymbol(getCandidateAlias()) returned null! getCandidateAlias()==\"" + getCandidateAlias() + "\"");
382    
383                            return expressionEvaluator.queryResultObjects(new ResultDescriptor(resultSymbol, null));
384                    }
385            }
386    
387            private AbstractExpressionEvaluator<?> expressionEvaluator;
388    
389            public AbstractExpressionEvaluator<?> getExpressionEvaluator() {
390                    return expressionEvaluator;
391            }
392    
393            private AbstractExpressionEvaluator<?> createExpressionEvaluatorTree(Expression expression)
394            {
395                    return createExpressionEvaluatorTreeRecursive(null, expression);
396            }
397    
398            private AbstractExpressionEvaluator<?> createExpressionEvaluatorTreeRecursive(AbstractExpressionEvaluator<?> parent, Expression expression)
399            {
400                    AbstractExpressionEvaluator<?> eval = createExpressionEvaluator(parent, expression);
401    
402                    if (expression.getLeft() != null) {
403                            AbstractExpressionEvaluator<?> childEval = createExpressionEvaluatorTreeRecursive(eval, expression.getLeft());
404                            eval.setLeft(childEval);
405                    }
406    
407                    if (expression.getRight() != null) {
408                            AbstractExpressionEvaluator<?> childEval = createExpressionEvaluatorTreeRecursive(eval, expression.getRight());
409                            eval.setRight(childEval);
410                    }
411    
412                    return eval;
413            }
414    
415            private AbstractExpressionEvaluator<?> createExpressionEvaluator(
416                            AbstractExpressionEvaluator<?> parent,
417                            Expression expr
418            )
419            {
420                    if (expr instanceof DyadicExpression) {
421                            DyadicExpression expression = (DyadicExpression) expr;
422                            if (
423                                            Expression.OP_EQ == expression.getOperator() ||
424                                            Expression.OP_NOTEQ == expression.getOperator() ||
425                                            Expression.OP_LT == expression.getOperator() ||
426                                            Expression.OP_LTEQ == expression.getOperator() ||
427                                            Expression.OP_GT == expression.getOperator() ||
428                                            Expression.OP_GTEQ == expression.getOperator()
429                            )
430                                    return new ComparisonExpressionEvaluator(this, parent, expression);
431                            else if (Expression.OP_AND == expression.getOperator())
432                                    return new AndExpressionEvaluator(this, parent, expression);
433                            else if (Expression.OP_OR == expression.getOperator())
434                                    return new OrExpressionEvaluator(this, parent, expression);
435                            else if (Expression.OP_NOT == expression.getOperator())
436                                    return new NotExpressionEvaluator(this, parent, expression);
437                            else
438                                    throw new UnsupportedOperationException("Unsupported operator for DyadicExpression: " + expr);
439                    }
440    
441                    if (expr instanceof PrimaryExpression)
442                            return new PrimaryExpressionEvaluator(this, parent, (PrimaryExpression) expr);
443    
444                    if (expr instanceof ParameterExpression)
445                            return new ParameterExpressionEvaluator(this, parent, (ParameterExpression) expr);
446    
447                    if (expr instanceof Literal)
448                            return new LiteralEvaluator(this, parent, (Literal) expr);
449    
450                    if (expr instanceof InvokeExpression)
451                            return new InvokeExpressionEvaluator(this, parent, (InvokeExpression) expr);
452    
453                    if (expr instanceof VariableExpression)
454                            return new VariableExpressionEvaluator(this, parent, (VariableExpression) expr);
455    
456                    if (expr instanceof SubqueryExpression)
457                            return new SubqueryExpressionEvaluator(this, parent, (SubqueryExpression) expr);
458    
459                    throw new UnsupportedOperationException("Don't know what to do with this expression: " + expr);
460            }
461    
462            public Object getObjectForDataEntry(DataEntry dataEntry)
463            {
464                    return getObjectForClassMetaAndObjectIDString(dataEntry.getClassMeta(), dataEntry.getObjectID());
465            }
466    
467            public Object getObjectForClassMetaAndObjectIDString(ClassMeta classMeta, String objectIDString)
468            {
469                    AbstractClassMetaData cmd = classMeta.getDataNucleusClassMetaData(ec);
470                    return IdentityUtils.getObjectFromIdString(objectIDString, cmd, ec, true);
471            }
472    
473            public Set<Long> getAllDataEntryIDsForCandidateClasses(Set<ClassMeta> candidateClassMetas)
474            {
475                    javax.jdo.Query q = getPersistenceManagerForData().newQuery(DataEntry.class);
476                    q.setResult("this.dataEntryID");
477    
478    
479                    Map<String, Object> queryParams = new HashMap<String, Object>();
480                    StringBuilder filter = new StringBuilder();
481    
482                    filter.append("this.keyStoreRefID == :keyStoreRefID && ");
483                    queryParams.put("keyStoreRefID", cryptoContext.getKeyStoreRefID());
484    
485                    filter.append(ClassMetaDAO.getMultiClassMetaOrFilterPart(queryParams, candidateClassMetas));
486    
487                    q.setFilter(filter.toString());
488    
489                    @SuppressWarnings("unchecked")
490                    Collection<Long> allDataEntryIDs = (Collection<Long>) q.executeWithMap(queryParams);
491                    Set<Long> result = new HashSet<Long>(allDataEntryIDs);
492                    q.closeAll();
493                    return result;
494            }
495    
496            public CryptoContext getCryptoContext() {
497                    return cryptoContext;
498            }
499    }