1 package org.wikimedia.search.extra;
2
3 import static java.util.Arrays.asList;
4 import static java.util.Collections.singletonList;
5 import static java.util.Collections.unmodifiableSet;
6
7 import java.io.IOException;
8 import java.util.Arrays;
9 import java.util.Collection;
10 import java.util.Collections;
11 import java.util.HashSet;
12 import java.util.List;
13 import java.util.Map;
14 import java.util.function.Supplier;
15
16 import org.elasticsearch.action.ActionRequest;
17 import org.elasticsearch.action.ActionResponse;
18 import org.elasticsearch.client.Client;
19 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
20 import org.elasticsearch.cluster.node.DiscoveryNodes;
21 import org.elasticsearch.cluster.service.ClusterService;
22 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
23 import org.elasticsearch.common.settings.ClusterSettings;
24 import org.elasticsearch.common.settings.IndexScopedSettings;
25 import org.elasticsearch.common.settings.Settings;
26 import org.elasticsearch.common.settings.SettingsFilter;
27 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
28 import org.elasticsearch.env.Environment;
29 import org.elasticsearch.env.NodeEnvironment;
30 import org.elasticsearch.index.IndexModule;
31 import org.elasticsearch.index.analysis.PreConfiguredTokenFilter;
32 import org.elasticsearch.index.analysis.TokenFilterFactory;
33 import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
34 import org.elasticsearch.monitor.os.OsService;
35 import org.elasticsearch.plugins.ActionPlugin;
36 import org.elasticsearch.plugins.AnalysisPlugin;
37 import org.elasticsearch.plugins.Plugin;
38 import org.elasticsearch.plugins.ScriptPlugin;
39 import org.elasticsearch.plugins.SearchPlugin;
40 import org.elasticsearch.repositories.RepositoriesService;
41 import org.elasticsearch.rest.RestController;
42 import org.elasticsearch.rest.RestHandler;
43 import org.elasticsearch.script.ScriptContext;
44 import org.elasticsearch.script.ScriptEngine;
45 import org.elasticsearch.script.ScriptService;
46 import org.elasticsearch.threadpool.ThreadPool;
47 import org.elasticsearch.watcher.ResourceWatcherService;
48 import org.wikimedia.search.extra.analysis.filters.PreserveOriginalFilter;
49 import org.wikimedia.search.extra.analysis.filters.TermFreqTokenFilter;
50 import org.wikimedia.search.extra.analysis.filters.TermFreqTokenFilterFactory;
51 import org.wikimedia.search.extra.fuzzylike.FuzzyLikeThisQueryBuilder;
52 import org.wikimedia.search.extra.latency.LatencyStatsAction;
53 import org.wikimedia.search.extra.latency.RestGetLatencyStats;
54 import org.wikimedia.search.extra.latency.SearchLatencyListener;
55 import org.wikimedia.search.extra.latency.TransportLatencyStatsAction;
56 import org.wikimedia.search.extra.levenshtein.LevenshteinDistanceScoreBuilder;
57 import org.wikimedia.search.extra.regex.SourceRegexQueryBuilder;
58 import org.wikimedia.search.extra.router.DegradedRouterQueryBuilder;
59 import org.wikimedia.search.extra.router.SystemLoad;
60 import org.wikimedia.search.extra.router.TokenCountRouterQueryBuilder;
61 import org.wikimedia.search.extra.simswitcher.SimSwitcherQueryBuilder;
62 import org.wikimedia.search.extra.superdetectnoop.ChangeHandler;
63 import org.wikimedia.search.extra.superdetectnoop.MultiListHandler;
64 import org.wikimedia.search.extra.superdetectnoop.SetHandler;
65 import org.wikimedia.search.extra.superdetectnoop.SuperDetectNoopScript;
66 import org.wikimedia.search.extra.superdetectnoop.VersionedDocumentHandler;
67 import org.wikimedia.search.extra.superdetectnoop.WithinAbsoluteHandler;
68 import org.wikimedia.search.extra.superdetectnoop.WithinPercentageHandler;
69 import org.wikimedia.search.extra.termfreq.TermFreqFilterQueryBuilder;
70 import org.wikimedia.search.extra.util.Suppliers.MutableSupplier;
71
72
73
74
75
76 @SuppressWarnings("classfanoutcomplexity")
77 public class ExtraCorePlugin extends Plugin implements SearchPlugin, AnalysisPlugin, ScriptPlugin, ActionPlugin {
78
79 private final SearchLatencyListener latencyListener;
80 private final MutableSupplier<ThreadPool> threadPoolSupplier;
81 private final SystemLoad loadStats;
82 private final SuperDetectNoopScript.SuperNoopScriptEngineService superDetectNoopService;
83
84 public ExtraCorePlugin(Settings settings) {
85 threadPoolSupplier = new MutableSupplier<>();
86 latencyListener = new SearchLatencyListener(threadPoolSupplier);
87 try {
88 loadStats = new SystemLoad(latencyListener, new OsService(settings));
89 } catch (IOException e) {
90 throw new RuntimeException("Couldn't init OsService", e);
91 }
92 superDetectNoopService = new SuperDetectNoopScript.SuperNoopScriptEngineService(
93 unmodifiableSet(new HashSet<>(asList(
94 new ChangeHandler.Equal.Recognizer(),
95 new WithinPercentageHandler.Recognizer(),
96 new WithinAbsoluteHandler.Recognizer(),
97 new SetHandler.Recognizer(),
98 new VersionedDocumentHandler.Recognizer(),
99 MultiListHandler.RECOGNIZER)
100 )));
101 }
102
103 @Override
104 public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
105 ResourceWatcherService resourceWatcherService, ScriptService scriptService,
106 NamedXContentRegistry xContentRegistry, Environment environment,
107 NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry,
108 IndexNameExpressionResolver indexNameExpressionResolver,
109 Supplier<RepositoriesService> repositoriesServiceSupplier
110 ) {
111 threadPoolSupplier.set(threadPool);
112 return singletonList(latencyListener);
113 }
114
115
116
117
118 @Override
119 @SuppressWarnings("deprecation")
120 public List<QuerySpec<?>> getQueries() {
121 return asList(
122 new QuerySpec<>(SourceRegexQueryBuilder.NAME, SourceRegexQueryBuilder::new, SourceRegexQueryBuilder::fromXContent),
123 new QuerySpec<>(FuzzyLikeThisQueryBuilder.NAME, FuzzyLikeThisQueryBuilder::new, FuzzyLikeThisQueryBuilder::fromXContent),
124 new QuerySpec<>(TokenCountRouterQueryBuilder.NAME, TokenCountRouterQueryBuilder::new, TokenCountRouterQueryBuilder::fromXContent),
125 new QuerySpec<>(DegradedRouterQueryBuilder.NAME,
126 in -> new DegradedRouterQueryBuilder(in, loadStats),
127 pc -> DegradedRouterQueryBuilder.fromXContent(pc, loadStats)),
128 new QuerySpec<>(SimSwitcherQueryBuilder.NAME, SimSwitcherQueryBuilder::new, SimSwitcherQueryBuilder::fromXContent),
129 new QuerySpec<>(TermFreqFilterQueryBuilder.NAME, TermFreqFilterQueryBuilder::new, TermFreqFilterQueryBuilder::fromXContent)
130 );
131 }
132
133 @Override
134 public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
135 return Collections.singletonMap("term_freq", AnalysisPlugin.requiresAnalysisSettings(TermFreqTokenFilterFactory::new));
136 }
137
138 @Override
139 public List<PreConfiguredTokenFilter> getPreConfiguredTokenFilters() {
140 return Arrays.asList(
141 PreConfiguredTokenFilter.singleton("preserve_original", true, PreserveOriginalFilter::new),
142 PreConfiguredTokenFilter.singleton("preserve_original_recorder", true, PreserveOriginalFilter.Recorder::new),
143 PreConfiguredTokenFilter.singleton("term_freq", true, TermFreqTokenFilter::new)
144 );
145 }
146
147 @Override
148 public ScriptEngine getScriptEngine(Settings settings, Collection<ScriptContext<?>> contexts) {
149 return superDetectNoopService;
150 }
151
152 @Override
153 public List<ScoreFunctionSpec<?>> getScoreFunctions() {
154 return singletonList(
155 new ScoreFunctionSpec<>(
156 LevenshteinDistanceScoreBuilder.NAME,
157 LevenshteinDistanceScoreBuilder::new,
158 LevenshteinDistanceScoreBuilder::fromXContent
159 )
160 );
161 }
162
163 @Override
164 public void onIndexModule(IndexModule indexModule) {
165 indexModule.addSearchOperationListener(latencyListener);
166 }
167
168 @Override
169 public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
170 return singletonList(
171 new ActionHandler<>(LatencyStatsAction.INSTANCE, TransportLatencyStatsAction.class)
172 );
173 }
174
175 @Override
176 public List<RestHandler> getRestHandlers(Settings settings, RestController restController,
177 ClusterSettings clusterSettings, IndexScopedSettings indexScopedSettings,
178 SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver,
179 Supplier<DiscoveryNodes> nodesInCluster) {
180 return singletonList(new RestGetLatencyStats());
181 }
182 }