DSL initial commit

dsl/examples/README.md

@@ -0,0 +1,17 @@
+# Examples
+
+In this folder you will find different examples to show the usage of the DSL.
+
+## Instructions
+Befoire to run any of the examples in this folder you should run `npm install` for installing all the required dependenices and the run the `loadRepo` script.
+
+## Run an example
+Running an example is very easy, you just need to run the following command:
+```sh
+npm run example examples/<filename>
+```
+
+For example:
+```sh
+npm run example examples/last-commits.ts
+```

dsl/examples/boolean-logic.ts

@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Client } from '../../'
+import { Q } from '../'
+
+async function run () {
+  const client = new Client({ node: 'http://localhost:9200' })
+
+  // define the query clauses
+  const fixDescription = Q.must(Q.match('description', 'fix'))
+  const files = Q.should(Q.term('files', 'test'), Q.term('files', 'docs'))
+  const author = Q.filter(Q.term('author.name', Q.param('author')))
+  const { body } = await client.search({
+    index: 'git',
+    // use the boolean utilities to craft the final query
+    body: Q.and(fixDescription, files, author)
+  })
+
+  console.log(body.hits.hits)
+}
+
+run().catch(console.log)

dsl/examples/compile-query.ts

@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Client } from '../../'
+import { Q } from '../'
+
+// You can compile a query if you need to get
+// the best performances out of your code.
+// The query crafting and compilation should be done
+// outside of your hot code path.
+// First of all yu should create your query almost
+// in the same way as you were doing before, the only
+// difference, is that all the paramegers you are passing
+// now should be updated with the `Q.param` API.
+// The only parameter or `Q.param`, is the name of the parameter
+// that you were passing before.
+const query = Q(
+  Q.match('description', Q.param('description')),
+  Q.filter(
+    Q.term('author.name', Q.param('author'))
+  ),
+  Q.size(10)
+)
+
+// Afterwards, you can create an interface that represents
+// the input object of the compiled query. The input object
+// contains all the parameters you were passing before, the
+// keys are the same you have passed to the various `Q.param`
+// invocations before. It defaults to `unknown`.
+interface Input {
+  description: string
+  author: string
+}
+// Once you have created the query and the input interface,
+// you must pass the  query to `Q.compile` and store the result
+// in a variable. `Q.compile` returns a function that accepts
+// a single object parameter, which is the same you have declared
+// in the interface before.
+const compiledQuery = Q.compile<Input>(query)
+
+async function run () {
+  const client = new Client({ node: 'http://localhost:9200' })
+
+  const { body } = await client.search({
+    index: 'git',
+    // Finally, you call the function inside your hot code path,
+    // the returned value will be the query.
+    body: compiledQuery({
+      description: 'fix',
+      author: 'delvedor'
+    })
+  })
+
+  console.log(body.hits.hits)
+}
+
+run().catch(console.log)

dsl/examples/day-most-commits.ts

@@ -0,0 +1,44 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Client } from '../../'
+import { Q, A } from '../'
+
+async function run () {
+  const client = new Client({ node: 'http://localhost:9200' })
+
+  // get the day where the most commits were made
+  const { body } = await client.search({
+    index: 'git',
+    body: Q(
+      Q.size(0),
+      // 'day_most_commits' is the name of the aggregation
+      A(A.day_most_commits.dateHistogram({
+        field: 'committed_date',
+        interval: 'day',
+        min_doc_count: 1,
+        order: { _count: 'desc' }
+      }))
+    )
+  })
+
+  console.log(body.aggregations)
+}
+
+run().catch(console.log)

dsl/examples/extend-aggregations.ts

@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Client } from '../../'
+import { Q, A } from '../'
+
+async function run () {
+  const client = new Client({ node: 'http://localhost:9200' })
+
+  // 'committers' is the name of the aggregation
+  let committersAgg = A.committers.terms('committer.name.keyword')
+  // instead of pass other aggregations as parameter
+  // to the parent aggregation, you can conditionally add them
+  if (Math.random() >= 0.5) {
+    committersAgg = A.committers.aggs(
+      committersAgg, A.line_stats.stats('stat.insertions')
+    )
+  }
+
+  const { body } = await client.search({
+    index: 'git',
+    body: Q(
+      Q.size(0),
+      A(committersAgg)
+    )
+  })
+
+  console.log(body.aggregations)
+}
+
+run().catch(console.log)

dsl/examples/extend-queries.ts

@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Client } from '../../'
+import { Q } from '../'
+
+async function run () {
+  const client = new Client({ node: 'http://localhost:9200' })
+
+  // the result must be fixes done by delvedor
+  let query = Q.bool(
+    Q.must(Q.match('description', 'fix')),
+    Q.filter(Q.term('author.name', 'delvedor'))
+  )
+
+  // Based on a condition, we want to enrich our query
+  if (Math.random() >= 0.5) {
+    // the results must be fixes done by delvedor
+    // on test or do files
+    const should = Q.should(
+      Q.term('files', 'test'),
+      Q.term('files', 'docs')
+    )
+    // The code below produces the same as the one above
+    // If you need to check multiple values for the same key,
+    // you can pass an array of strings instead of calling
+    // the query function multiple times
+    // ```
+    //   const should = Q.should(
+    //    Q.term('files', ['test', 'docs'])
+    //   )
+    // ```
+    query = Q.and(query, should)
+  } else {
+    // the results must be fixes or features done by delvedor
+    const must = Q.must(
+      Q.match('description', 'feature')
+    )
+    query = Q.or(query, must)
+  }
+
+  const { body } = await client.search({
+    index: 'git',
+    body: query
+  })
+
+  console.log(body.hits.hits)
+}
+
+run().catch(console.log)

dsl/examples/fix-commit.ts

@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Client } from '../../'
+import { Q } from '../'
+
+async function run () {
+  const client = new Client({ node: 'http://localhost:9200' })
+
+  // search commits that contains 'fix' but do not changes test files
+  const { body } = await client.search({
+    index: 'git',
+    body: Q.bool(
+      // You can avoid to call `Q.must`, as any query will be
+      // sent inside a `must` block unless specified otherwise
+      Q.must(Q.match('description', 'fix')),
+      Q.mustNot(Q.term('files', 'test'))
+    )
+  })
+
+  console.log(body.hits.hits)
+}
+
+run().catch(console.log)

dsl/examples/last-commits.ts

@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Client } from '../../'
+import { Q } from '../'
+
+async function run () {
+  const client = new Client({ node: 'http://localhost:9200' })
+
+  // last 10 commits for 'elasticsearch-js' repo
+  const { body } = await client.search({
+    index: 'git',
+    body: Q(
+      Q.term('repository', 'elasticsearch-js'),
+      Q.sort('committed_date', { order: 'desc' }),
+      Q.size(10)
+    )
+  })
+
+  console.log(body.hits.hits)
+}
+
+run().catch(console.log)

dsl/examples/loadRepo.js

@@ -0,0 +1,159 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+'use strict'
+
+const minimist = require('minimist')
+const Git = require('simple-git/promise')
+const { Client } = require('@elastic/elasticsearch')
+
+start(minimist(process.argv.slice(2), {
+  string: ['elasticsearch', 'index', 'repository'],
+  default: {
+    elasticsearch: 'http://localhost:9200',
+    index: 'git',
+    repository: 'elasticsearch-js'
+  }
+}))
+
+async function start ({ elasticsearch, index, repository }) {
+  const client = new Client({ node: elasticsearch })
+  await createIndex({ client, index })
+  await loadHistory({ client, index, repository })
+}
+
+async function createIndex ({ client, index }) {
+  const userMapping = {
+    properties: {
+      name: {
+        type: 'text',
+        fields: {
+          keyword: { type: 'keyword' }
+        }
+      }
+    }
+  }
+
+  await client.indices.create({
+    index,
+    body: {
+      settings: {
+        // just one shard, no replicas for testing
+        number_of_shards: 1,
+        number_of_replicas: 0,
+        // custom analyzer for analyzing file paths
+        analysis: {
+          analyzer: {
+            file_path: {
+              type: 'custom',
+              tokenizer: 'path_hierarchy',
+              filter: ['lowercase']
+            }
+          }
+        }
+      },
+      mappings: {
+        properties: {
+          repository: { type: 'keyword' },
+          sha: { type: 'keyword' },
+          author: userMapping,
+          authored_date: { type: 'date' },
+          committer: userMapping,
+          committed_date: { type: 'date' },
+          parent_shas: { type: 'keyword' },
+          description: { type: 'text', analyzer: 'snowball' },
+          files: { type: 'text', analyzer: 'file_path', fielddata: true }
+        }
+      }
+    }
+  })
+}
+
+async function loadHistory ({ client, index, repository }) {
+  const git = Git(process.cwd())
+  // Get the result of  'git log'
+  const { all: history } = await git.log({
+    format: {
+      hash: '%H',
+      parentHashes: '%P',
+      authorName: '%an',
+      authorEmail: '%ae',
+      authorDate: '%ai',
+      committerName: '%cn',
+      committerEmail: '%ce',
+      committerDate: '%cd',
+      subject: '%s'
+    }
+  })
+
+  // Get the stats for every commit
+  for (var i = 0; i < history.length; i++) {
+    const commit = history[i]
+    const stat = await git.show(['--numstat', '--oneline', commit.hash])
+    commit.files = []
+    commit.stat = stat
+      .split('\n')
+      .slice(1)
+      .filter(Boolean)
+      .reduce((acc, val, index) => {
+        const [insertions, deletions, file] = val.split('\t')
+        commit.files.push(file)
+        acc.files++
+        acc.insertions += Number(insertions)
+        acc.deletions += Number(deletions)
+        return acc
+      }, { insertions: 0, deletions: 0, files: 0 })
+  }
+
+  // Index the data, 500 commits at a time
+  var count = 0
+  var chunk = history.slice(count, count + 500)
+  while (chunk.length > 0) {
+    const { body } = await client.bulk({
+      body: chunk.reduce((body, commit) => {
+        body.push({ index: { _index: index, _id: commit.hash } })
+        body.push({
+          repository,
+          sha: commit.hash,
+          author: {
+            name: commit.authorName,
+            email: commit.authorEmail
+          },
+          authored_date: new Date(commit.authorDate).toISOString(),
+          committer: {
+            name: commit.committerName,
+            email: commit.committerEmail
+          },
+          committed_date: new Date(commit.committerDate).toISOString(),
+          parent_shas: commit.parentHashes,
+          description: commit.subject,
+          files: commit.files,
+          stat: commit.stat
+        })
+        return body
+      }, [])
+    })
+    if (body.errors) {
+      console.log(JSON.stringify(body.items[0], null, 2))
+      process.exit(1)
+    }
+    count += 500
+    chunk = history.slice(count, count + 500)
+  }
+}

dsl/examples/top-committers.ts

@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Client } from '../../'
+import { Q, A } from '../'
+
+async function run () {
+  const client = new Client({ node: 'http://localhost:9200' })
+
+  // top committers aggregation
+  // 'committers' is the name of the aggregation
+  const committersAgg = A.committers.terms(
+    { field: 'committer.name.keyword' },
+    // you can nest multiple aggregations by
+    // passing them to the aggregation constructor
+    // 'line_stats' is the name of the aggregation
+    A.line_stats.stats({ field: 'stat.insertions' })
+  )
+  const { body } = await client.search({
+    index: 'git',
+    body: Q(
+      Q.matchAll(),
+      Q.size(0),
+      A(committersAgg)
+    )
+  })
+
+  console.log(body.aggregations)
+}
+
+run().catch(console.log)

dsl/examples/top-month.ts

@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Client } from '../../'
+import { Q, A } from '../'
+
+async function run () {
+  const client = new Client({ node: 'http://localhost:9200' })
+
+  const committers = A.committers.terms(
+    { field: 'committer.name.keyword' },
+    A.insertions.sum({ field: 'stat.insertions' })
+  )
+  const topCommittersPerMonth = A.top_committer_per_month.maxBucket(
+    { bucket_path: 'committers>insertions' }
+  )
+  const commitsPerMonth = A.commits_per_month.dateHistogram(
+    {
+      field: 'committed_date',
+      interval: 'day',
+      min_doc_count: 1,
+      order: { _count: 'desc' }
+    },
+    // nested aggregations
+    committers,
+    topCommittersPerMonth
+  )
+  const topCommittersPerMonthGlobal = A.top_committer_per_month.maxBucket(
+    { bucket_path: 'commits_per_month>top_committer_per_month' }
+  )
+
+  const { body: topMonths } = await client.search({
+    index: 'git',
+    body: Q(
+      // we want to know the top month for 'delvedor'
+      Q.filter(Q.term('author', 'delvedor')),
+      Q.size(0),
+      A(commitsPerMonth, topCommittersPerMonthGlobal)
+    )
+  })
+
+  console.log(topMonths)
+}
+
+run().catch(console.log)