View Javadoc
1   package org.wikimedia.search.extra;
2   
3   import static java.util.Arrays.asList;
4   import static java.util.Collections.singletonList;
5   import static java.util.Collections.unmodifiableSet;
6   
7   import java.io.IOException;
8   import java.util.Arrays;
9   import java.util.Collection;
10  import java.util.Collections;
11  import java.util.HashSet;
12  import java.util.List;
13  import java.util.Map;
14  import java.util.function.Supplier;
15  
16  import org.elasticsearch.action.ActionRequest;
17  import org.elasticsearch.action.ActionResponse;
18  import org.elasticsearch.client.Client;
19  import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
20  import org.elasticsearch.cluster.node.DiscoveryNodes;
21  import org.elasticsearch.cluster.service.ClusterService;
22  import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
23  import org.elasticsearch.common.settings.ClusterSettings;
24  import org.elasticsearch.common.settings.IndexScopedSettings;
25  import org.elasticsearch.common.settings.Settings;
26  import org.elasticsearch.common.settings.SettingsFilter;
27  import org.elasticsearch.common.xcontent.NamedXContentRegistry;
28  import org.elasticsearch.env.Environment;
29  import org.elasticsearch.env.NodeEnvironment;
30  import org.elasticsearch.index.IndexModule;
31  import org.elasticsearch.index.analysis.PreConfiguredTokenFilter;
32  import org.elasticsearch.index.analysis.TokenFilterFactory;
33  import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
34  import org.elasticsearch.monitor.os.OsService;
35  import org.elasticsearch.plugins.ActionPlugin;
36  import org.elasticsearch.plugins.AnalysisPlugin;
37  import org.elasticsearch.plugins.Plugin;
38  import org.elasticsearch.plugins.ScriptPlugin;
39  import org.elasticsearch.plugins.SearchPlugin;
40  import org.elasticsearch.repositories.RepositoriesService;
41  import org.elasticsearch.rest.RestController;
42  import org.elasticsearch.rest.RestHandler;
43  import org.elasticsearch.script.ScriptContext;
44  import org.elasticsearch.script.ScriptEngine;
45  import org.elasticsearch.script.ScriptService;
46  import org.elasticsearch.threadpool.ThreadPool;
47  import org.elasticsearch.watcher.ResourceWatcherService;
48  import org.wikimedia.search.extra.analysis.filters.PreserveOriginalFilter;
49  import org.wikimedia.search.extra.analysis.filters.TermFreqTokenFilter;
50  import org.wikimedia.search.extra.analysis.filters.TermFreqTokenFilterFactory;
51  import org.wikimedia.search.extra.fuzzylike.FuzzyLikeThisQueryBuilder;
52  import org.wikimedia.search.extra.latency.LatencyStatsAction;
53  import org.wikimedia.search.extra.latency.RestGetLatencyStats;
54  import org.wikimedia.search.extra.latency.SearchLatencyListener;
55  import org.wikimedia.search.extra.latency.TransportLatencyStatsAction;
56  import org.wikimedia.search.extra.levenshtein.LevenshteinDistanceScoreBuilder;
57  import org.wikimedia.search.extra.regex.SourceRegexQueryBuilder;
58  import org.wikimedia.search.extra.router.DegradedRouterQueryBuilder;
59  import org.wikimedia.search.extra.router.SystemLoad;
60  import org.wikimedia.search.extra.router.TokenCountRouterQueryBuilder;
61  import org.wikimedia.search.extra.simswitcher.SimSwitcherQueryBuilder;
62  import org.wikimedia.search.extra.superdetectnoop.ChangeHandler;
63  import org.wikimedia.search.extra.superdetectnoop.MultiListHandler;
64  import org.wikimedia.search.extra.superdetectnoop.SetHandler;
65  import org.wikimedia.search.extra.superdetectnoop.SuperDetectNoopScript;
66  import org.wikimedia.search.extra.superdetectnoop.VersionedDocumentHandler;
67  import org.wikimedia.search.extra.superdetectnoop.WithinAbsoluteHandler;
68  import org.wikimedia.search.extra.superdetectnoop.WithinPercentageHandler;
69  import org.wikimedia.search.extra.termfreq.TermFreqFilterQueryBuilder;
70  import org.wikimedia.search.extra.util.Suppliers.MutableSupplier;
71  
72  
73  /**
74   * Setup the Elasticsearch plugin.
75   */
76  @SuppressWarnings("classfanoutcomplexity")
77  public class ExtraCorePlugin extends Plugin implements SearchPlugin, AnalysisPlugin, ScriptPlugin, ActionPlugin {
78  
79      private final SearchLatencyListener latencyListener;
80      private final MutableSupplier<ThreadPool> threadPoolSupplier;
81      private final SystemLoad loadStats;
82      private final SuperDetectNoopScript.SuperNoopScriptEngineService superDetectNoopService;
83  
84      public ExtraCorePlugin(Settings settings) {
85          threadPoolSupplier = new MutableSupplier<>();
86          latencyListener = new SearchLatencyListener(threadPoolSupplier);
87          try {
88              loadStats = new SystemLoad(latencyListener, new OsService(settings));
89          } catch (IOException e) {
90              throw new RuntimeException("Couldn't init OsService", e);
91          }
92          superDetectNoopService = new SuperDetectNoopScript.SuperNoopScriptEngineService(
93                  unmodifiableSet(new HashSet<>(asList(
94                      new ChangeHandler.Equal.Recognizer(),
95                      new WithinPercentageHandler.Recognizer(),
96                      new WithinAbsoluteHandler.Recognizer(),
97                      new SetHandler.Recognizer(),
98                      new VersionedDocumentHandler.Recognizer(),
99                      MultiListHandler.RECOGNIZER)
100         )));
101     }
102 
103     @Override
104     public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
105                                                ResourceWatcherService resourceWatcherService, ScriptService scriptService,
106                                                NamedXContentRegistry xContentRegistry, Environment environment,
107                                                NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry,
108                                                IndexNameExpressionResolver indexNameExpressionResolver,
109                                                Supplier<RepositoriesService> repositoriesServiceSupplier
110     ) {
111         threadPoolSupplier.set(threadPool);
112         return singletonList(latencyListener);
113     }
114 
115     /**
116      * Register our parsers.
117      */
118     @Override
119     @SuppressWarnings("deprecation")
120     public List<QuerySpec<?>> getQueries() {
121         return asList(
122                 new QuerySpec<>(SourceRegexQueryBuilder.NAME, SourceRegexQueryBuilder::new, SourceRegexQueryBuilder::fromXContent),
123                 new QuerySpec<>(FuzzyLikeThisQueryBuilder.NAME, FuzzyLikeThisQueryBuilder::new, FuzzyLikeThisQueryBuilder::fromXContent),
124                 new QuerySpec<>(TokenCountRouterQueryBuilder.NAME, TokenCountRouterQueryBuilder::new, TokenCountRouterQueryBuilder::fromXContent),
125                 new QuerySpec<>(DegradedRouterQueryBuilder.NAME,
126                         in -> new DegradedRouterQueryBuilder(in, loadStats),
127                         pc -> DegradedRouterQueryBuilder.fromXContent(pc, loadStats)),
128                 new QuerySpec<>(SimSwitcherQueryBuilder.NAME, SimSwitcherQueryBuilder::new, SimSwitcherQueryBuilder::fromXContent),
129                 new QuerySpec<>(TermFreqFilterQueryBuilder.NAME, TermFreqFilterQueryBuilder::new, TermFreqFilterQueryBuilder::fromXContent)
130         );
131     }
132 
133     @Override
134     public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
135         return Collections.singletonMap("term_freq", AnalysisPlugin.requiresAnalysisSettings(TermFreqTokenFilterFactory::new));
136     }
137 
138     @Override
139     public List<PreConfiguredTokenFilter> getPreConfiguredTokenFilters() {
140         return Arrays.asList(
141             PreConfiguredTokenFilter.singleton("preserve_original", true, PreserveOriginalFilter::new),
142             PreConfiguredTokenFilter.singleton("preserve_original_recorder", true, PreserveOriginalFilter.Recorder::new),
143             PreConfiguredTokenFilter.singleton("term_freq", true, TermFreqTokenFilter::new)
144         );
145     }
146 
147     @Override
148     public ScriptEngine getScriptEngine(Settings settings, Collection<ScriptContext<?>> contexts) {
149         return superDetectNoopService;
150     }
151 
152     @Override
153     public List<ScoreFunctionSpec<?>> getScoreFunctions() {
154         return singletonList(
155             new ScoreFunctionSpec<>(
156                     LevenshteinDistanceScoreBuilder.NAME,
157                     LevenshteinDistanceScoreBuilder::new,
158                     LevenshteinDistanceScoreBuilder::fromXContent
159             )
160         );
161     }
162 
163     @Override
164     public void onIndexModule(IndexModule indexModule) {
165         indexModule.addSearchOperationListener(latencyListener);
166     }
167 
168     @Override
169     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
170         return singletonList(
171                 new ActionHandler<>(LatencyStatsAction.INSTANCE, TransportLatencyStatsAction.class)
172         );
173     }
174 
175     @Override
176     public List<RestHandler> getRestHandlers(Settings settings, RestController restController,
177                                              ClusterSettings clusterSettings, IndexScopedSettings indexScopedSettings,
178                                              SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver,
179                                              Supplier<DiscoveryNodes> nodesInCluster) {
180         return singletonList(new RestGetLatencyStats());
181     }
182 }