diff --git a/plugins/tntsearch/.eslintrc b/plugins/tntsearch/.eslintrc new file mode 100644 index 0000000..79ecb4a --- /dev/null +++ b/plugins/tntsearch/.eslintrc @@ -0,0 +1,15 @@ +{ + "root": true, + "extends": "defaults/configurations/airbnb/es6", + + "rules": { + "no-empty-label": 0, + "space-after-keywords": "off", + "space-return-throw-case": "off", + + "no-param-reassign": 0, + "indent": [2, 4, { "SwitchCase": 1 }], + "no-labels": 2, + "keyword-spacing": [2, {"before": true, "after": true}] + } +} \ No newline at end of file diff --git a/plugins/tntsearch/.gitignore b/plugins/tntsearch/.gitignore new file mode 100644 index 0000000..a0f2b01 --- /dev/null +++ b/plugins/tntsearch/.gitignore @@ -0,0 +1,2 @@ +/node_modules +/.idea diff --git a/plugins/tntsearch/CHANGELOG.md b/plugins/tntsearch/CHANGELOG.md new file mode 100644 index 0000000..94bdf5d --- /dev/null +++ b/plugins/tntsearch/CHANGELOG.md @@ -0,0 +1,220 @@ +# v3.4.0 +## 03/06/2023 + +1. [](#improved) + * Updated TNTSearch library to `2.9.0` + * Enable Fuzy search [#123](https://github.com/trilbymedia/grav-plugin-tntsearch/pull/123) + * Add configuration for Levenshtein distance for fuzzy search [#124](https://github.com/trilbymedia/grav-plugin-tntsearch/pull/124) + * Added French translation [#100](https://github.com/trilbymedia/grav-plugin-tntsearch/issues/100) + * Added missing stemmers [#115](https://github.com/trilbymedia/grav-plugin-tntsearch/pull/115) [#116](https://github.com/trilbymedia/grav-plugin-tntsearch/pull/116) + +# v3.3.1 +## 02/25/2021 + +1. [](#improved) + * Upgraded to TNTSearch version `2.6.0` + * Added German (de) language [#103](https://github.com/trilbymedia/grav-plugin-tntsearch/pull/103) +1. [](#bugfix) + * Fixed `query` truncation when containing a hash (`#`) and preventing proper search results [#110](https://github.com/trilbymedia/grav-plugin-tntsearch/pull/110) + * Fixed `q` query parameter not working [#111](https://github.com/trilbymedia/grav-plugin-tntsearch/pull/111) + * Fix default stemmer and description [#105](https://github.com/trilbymedia/grav-plugin-tntsearch/pull/105) + * Fixed PHP 8 compatibility issues + +# v3.3.0 +## 12/02/2020 + +1. [](#improved) + * Upgraded to TNTSearch version `2.5.0` + * Pass phpstan level 7 tests +1. [](#bugfix) + * Fixed FlexPages events for add+delete + * Fixed running scheduled index job [#104](https://github.com/trilbymedia/grav-plugin-tntsearch/pull/104) + +# v3.2.1 +## 09/04/2020 + +1. [](#bugfix) + * Fixed bad `require("history")...` JS warning [#101](https://github.com/trilbymedia/grav-plugin-tntsearch/issues/101) + +# v3.2.0 +## 06/08/2020 + +1. [](#new) + * Added support for CLI `bin/plugin index` to index only a single language (`--language=en`) +1. [](#improved) + * Renamed CLI classes to avoid class name conflicts +1. [](#bugfix) + * Fixed non-routable and non-published pages showing up in search results + * Fixed indexing in multi-language sites + * Use CLI command directly in scheduler command to work [#95](https://github.com/trilbymedia/grav-plugin-tntsearch/issues/95) + +# v3.1.1 +## 02/12/2020 + +1. [](#improved) + * Search with JS disabled [#75](https://github.com/trilbymedia/grav-plugin-tntsearch/pull/75) + * Added RU 🇷🇺 language [#74](https://github.com/trilbymedia/grav-plugin-tntsearch/pull/74) + * Various JS dependency updates & recompiled production JS +1. [](#bugfix) + * Added missing `search_object_type` to blueprint + +# v3.1.0 +## 02/11/2020 + +1. [](#new) + * Require Grav v1.6.21 + * Upgraded to TNTSearch version 2.2 (PHP 7.4 fixes) +1. [](#improved) + * Code cleanup +1. [](#bugfix) + * Fixed Grav initialization in CLI + * Work around inconsistencies in page content if page template uses `grav.page` instead of `page` + +# v3.0.1 +## 02/03/2020 + +1. [](#bugfix) + * Fixed an issue indexing via Admin with Grav 1.7 + +# v3.0.0 +## 04/14/2019 + +1. [](#new) + * Added new Grav Scheduler integration + * Added new Multi-Language Support +1. [](#improved) + * Switched to latest TNTSearch version 2.0 (PHP 7.1+) + * Added a new `onFlexObjecSave()` event + * Simplified indexing logic + * Code cleanup + * Minor CSS improvements for search field + * Implemented a unified indexer process that always uses the CLI command for consistency + * Use Grav YAML handler +1. [](#bugfix) + * Use custom search object in query [#63](https://github.com/trilbymedia/grav-plugin-tntsearch/issues/63) + * Fixed issue with Ajax results escaping + * Fixed issues when updating search index + * Set the db index file as a property of `GravTNTSearch` to allow for better overriding + * Put better type checking around `onTNTSearchIndex()` example that indexes `page.header.author` + +# v2.0.4 +## 09/21/2018 + +1. [](#new) + * Added new `tntsearch: index: true|false` page header option to skip specific pages +1. [](#bugfix) + * Skip indexing of pages with `redirect` set in page header [#21](https://github.com/trilbymedia/grav-plugin-tntsearch/issues/21) + +# v2.0.3 +## 08/16/2018 + +1. [](#new) + * New option to allow disabling of page events, manual updates will be required to pick up changes +1. [](#bugfix) + * Don't remove the X button if `built_in_css` is `false` + +# v2.0.2 +## 07/20/2018 + +1. [](#bugfix) + * Ensure that credentials are passed in when searching via `fetch` + * Compressed JS for better performance + +# v2.0.1 +## 05/21/2018 + +1. [](#bugfix) + * Potential fix for history conflicts. + +# v2.0.0 +## 05/11/2018 + +1. [](#new) + * Refactored TNTSearch to allow core classes to be extensible by other plugins + * Added `phrases` search support [#32](https://github.com/trilbymedia/grav-plugin-tntsearch/pull/32) +1. [](#improved) + * Defaulted TNTSearch to search **all pages** out of the box. This should be tweaked though + * Added auto-focus to search input [#28](https://github.com/trilbymedia/grav-plugin-tntsearch/pull/28) + * Added option to control `powered by` [#34](https://github.com/trilbymedia/grav-plugin-tntsearch/pull/34) + * Added a timer on CLI index command + * Exposing `GravTNTSearch` to the browser for JS manipulation + * Dispatching `tntsearch:start` and `tntsearch:done` events when starting/rendering results + * README.md typo fixes +1. [](#bugfix) + * Implemented options as default values that were being ignored + * Fixed missing `break` in foreach [#33](https://github.com/trilbymedia/grav-plugin-tntsearch/pull/33) + * Add missing `use` statement [#41](https://github.com/trilbymedia/grav-plugin-tntsearch/pull/41) + +# v1.2.5 +## 03/07/2018 + +1. [](#improved) + * Only update the a page on save if it exists in the current filter and is therefore eligible to be indexed\ + * Removed Admin dependency, it works fine without admin too, just need to use CLI + +# v1.2.4 +## 02/14/2018 + +1. [](#bugfix) + * Fix issue with admin saving 'string' for filter [#25](https://github.com/trilbymedia/grav-plugin-tntsearch/issues/25) + +# v1.2.3 +## 02/14/2018 + +1. [](#bugfix) + * Missing comma in Admin JS breaking quick-tray reindexing + +# v1.2.2 +## 02/09/2018 + +1. [](#improved) + * Updated TNTSearch to use version `1.3.1` of TNTSearch library for PHP 7.2 compatibility [#24](https://github.com/trilbymedia/grav-plugin-tntsearch/issues/24) +1. [](#bugfix) + * Fixed URI `hash` getting unintentionally removed by TNTSearch [#15](https://github.com/trilbymedia/grav-plugin-tntsearch/pull/15) + * Fixed issue with param separator needed for Windows [#16](https://github.com/trilbymedia/grav-plugin-tntsearch/pull/16) + * Fixed placeholder format in blueprint [#18](https://github.com/trilbymedia/grav-plugin-tntsearch/pull/18) + +# v1.2.1 +## 01/16/2018 + +1. [](#new) + * Added `onTNTSearchReIndex()` that you can fire from any plugin to reindex everything +1. [](#bugfix) + * Fixed an XSS exploit in query + +# v1.2.0 +## 10/29/2017 + +1. [](#new) + * Reworked JS to VanillaJS [#12](https://github.com/trilbymedia/grav-plugin-tntsearch/pull/12) + * Implemented live URI / history refresh when typing in the field + * Added new 'auto' setting for search_type that automatically detects 'basic' or 'boolean'. + * It is now possible to force a search_type mode whether it's `basic` or `boolean` + * Updated to TNTSearch Library to v1.1.0 +1. [](#improved) + * Allow the ability to pass a `placeholder` to the `partials/tntsearch.html.twig` template + * Moved 'fuzzy' option as independent option +1. [](#bugfix) + * Fixed JS issue when at login page + * Fixed results showing on load for drop-downs, instead of in_page only view [#10](https://github.com/trilbymedia/grav-plugin-tntsearch/issues/10) + +# v1.1.0 +## 08/22/2017 + +1. [](#new) + * Extensible output JSON support via new `onTTNTSearchQuery()` event. + * Added a 'powered-by' link that can be disabled via configuration + * Improved docs by including instructions on how to use CLI to index. + +# v1.0.1 +## 08/22/2017 + +1. [](#new) + * Changed cartoon bomb icon with more friendly version (binoculars) [#4](https://github.com/trilbymedia/grav-plugin-tntsearch/issues/4) + * Added the ability to disable CSS and JS independently [#3](https://github.com/trilbymedia/grav-plugin-tntsearch/issues/3) + +# v1.0.0 +## 08/16/2017 + +1. [](#new) + * Initial release... diff --git a/plugins/tntsearch/LICENSE b/plugins/tntsearch/LICENSE new file mode 100644 index 0000000..fa236f8 --- /dev/null +++ b/plugins/tntsearch/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017 Trilby Media, LLC + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/plugins/tntsearch/README.md b/plugins/tntsearch/README.md new file mode 100644 index 0000000..e9541ce --- /dev/null +++ b/plugins/tntsearch/README.md @@ -0,0 +1,373 @@ +# TNTSearch Plugin + +The **TNTSearch** Plugin is for [Grav CMS](http://github.com/getgrav/grav). Powerful indexed-based full text search engine powered by the [TNTSearch library](https://github.com/teamtnt/tntsearch) that provides fast Ajax-based Grav content searches. This plugin is highly flexible allowing indexes of arbitrary content data as well as custom Twig templates to provide the opportunity to index modular and other dynamic page types. TNTSearch provides CLI as well as Admin based administration and re-indexing, as well as a built-in Ajax-powered front-end search tool. + +> NOTE: TNTSearch version 3.0.0 now requires Grav 1.6.0 or newer to function as it makes use of new functionality not available in previous versions. + +![](assets/tntsearch-ajax.gif) + +## Installation + +Installing the Tnt Search plugin can be done in one of two ways. The GPM (Grav Package Manager) installation method enables you to quickly and easily install the plugin with a simple terminal command, while the manual method enables you to do so via a zip file. + +### GPM Installation (Preferred) + +The simplest way to install this plugin is via the [Grav Package Manager (GPM)](http://learn.getgrav.org/advanced/grav-gpm) through your system's terminal (also called the command line). From the root of your Grav install type: + + bin/gpm install tntsearch + +This will install the Tnt Search plugin into your `/user/plugins` directory within Grav. Its files can be found under `/your/site/grav/user/plugins/tntsearch`. + +## Requirements + +Other than standard Grav requirements, this plugin does have some extra requirements. Due to the complex nature of a search engine, TNTSearch utilizes a flat-file database to store its wordlist as well as the mapping for content. This is handled automatically by the plugin, but you do need to ensure you have the following installed on your server: + +* **SQLite3** Database +* **PHP pdo** Extension +* **PHP pdo_sqlite** Driver +* **PHP pdo_mysql** Driver (only required because library references some MySQL constants, MySQL db is not used) + +| PHP by default comes with **PDO** and the vast majority of linux-based systems already come with SQLite. + +### Installation of SQLite on Mac systems + +SQLite actually comes pre-installed on your Mac, but you can upgrade it to the latest version with Homebrew: + +Install [Homebrew](https://brew.sh/) + +```shell +$ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +``` + +Install SQLite with Homebrew + +```shell +$ brew install sqlite +``` + +### Installation of SQLite on Windows systems + +Download the appropriate version of SQLite from the [SQLite Downloads Page](https://www.sqlite.org/download.html). + +Extract the downloaded ZIP file and run the `sqlite3.exe` executable. + + +## Configuration + +Before configuring this plugin, you should copy the `user/plugins/tntsearch/tntsearch.yaml` to `user/config/plugins/tntsearch.yaml` and only edit that copy. + +Here is the default configuration and an explanation of available options: + +```yaml +enabled: true +search_route: '/search' +query_route: '/s' +built_in_css: true +built_in_js: true +built_in_search_page: true +enable_admin_page_events: true +search_type: auto +fuzzy: false +distance: 2 +phrases: true +stemmer: 'no' +display_route: true +display_hits: true +display_time: true +live_uri_update: true +limit: 20 +min: 3 +snippet: 300 +index_page_by_default: true +scheduled_index: + enabled: false + at: '0 */3 * * *' + logs: 'logs/tntsearch-index.out' +filter: + items: + - root@.descendants +powered_by: true +search_object_type: Grav +``` + +The configuration options are as follows: + +* `enabled` - enable or disable the plugin instantly +* `search_route` - the route used for the built-in search page +* `query_route` - the route used by the search form to query the search engine +* `built_in_css` - enable or disable the built-in css styling +* `built_in_js` - enable or disable the built-in javascript +* `built_in_search_page` - enable or disable the built-in search page +* `enable_admin_page_events` - enable or disable the page events which occur `on-save` to add/update/remove page in index +* `search_type` - can be one of these types: + * `basic` - standard string matching + * `boolean` - supports `or` or `minus`. e.g. `foo -bar` + * `auto` - automatically detects whether to use `basic` or `boolean` +* `fuzzy` - matches if the words are 'close' but not necessarily exact matches +* `distance` - Levenshtein distance of fuzzy search. It represents the amount of characters which need to be changed, removed, or added in a word in order it to match the search keyword. Increasing the distance produces more search results but decreases the accuracy of the search. +* `phrases` - automatically handle phrases support +* `stemmer` - can be one of these types: + * `no` - no stemmer + * `arabic` - Arabic language + * `croatian` - Croatian language + * `german` - German language + * `italian` - Italian language + * `porter` - Porter stemmer for English language + * `portuguese` - Portuguese language + * `russian` - Russian language + * `ukrainian` - Ukrainian language +* `display_route` - display the route in the search results +* `display_hits` - display the number of hits in the search results +* `display_time` - display the execution time in the search results +* `live_uri_update` - when `built_in_js` is enabled, live updates the URI bar in the `search_route` page +* `limit` - maximum amount of results to be shown +* `min` - mininum amount of characters typed before performing search +* `snippet` - amount of characters for previewing a result item +* `index_page_by_default` - should all pages be indexed by default unless frontmatter overrides +* `scheduled_index` - New scheduled index job. Disabled by default, when enabled defaulted to run every 3 hours, and output results to `logs/tntsearch-index.out` +* `filter` - a [Page Collections filter](https://learn.getgrav.org/content/collections#summary-of-collection-options) that lets you pick specific pages to index via a collection query +* `powered_by` - Display the **powered-by TNTSearch** text +* `search_object_type` - Allows custom classes to override the default **Grav Page** support. This allows completely custom searching capabilities for any data type. + +## Usage + +TNTSearch relies on your content being indexed into the SQLite index database before any search queries can be made. This is very similar to other search engines such as ElasticSearch, Solr, Lucene, etc, but it uses a relatively simply PHP search engine library [TNTSearch library](https://github.com/teamtnt/tntsearch) to achieve this with little setup and no hassles. + +### Indexing + +The first step after installation of the plugin, is to index your content. There are several ways you can accomplish this. + +#### CLI Indexing + +First if you are able to access the CLI or just choose not to use the admin plugin, you can use the built-in CLI command: + +```shell +$ bin/plugin tntsearch index +``` + +This will scan all your pages and index the content. You should see some output like this: + +```shell +Re-indexing Search + +Added 1 / +Added 2 /blog/classic-modern-architecture +Added 3 /blog/daring-fireball-link +Added 4 /blog/focus-and-blur +Added 5 /blog/just-some-text-today +Added 6 /blog/london-industry +Added 7 /blog/random-thoughts +Added 8 /blog/sunshine-in-the-hills +Added 9 /blog/the-urban-jungle +Total rows 9 +Done. +``` + +This indicates a successful indexing of your content. + +#### Admin Plugin Indexing + +If you are using the admin plugin you can index your content directly from the plugin. TNTSearch adds a new **quick-tray** icon that lets you create a new index or re-index all your content quickly and conveniently with a single click. + +![](assets/tntsearch-quicktray.png) + +Alternatively you can navigate to the TNTSearch configuration section and click the `Index Content` button: + +![](assets/tntsearch-config.png) + +#### Skipping Indexing + +> NOTE: That any page that uses a `redirect` page header attribute will be skipped during indexing. + +You can explicitly skip a page that is in the index filter by adding this YAML to the page header: + +``` +tntsearch: + index: false +``` + +#### Multi-Language Support + +With the new 3.0 version of TNTSearch, support has been added for multiple languages (Grav 1.6 required). Internally, this means that rather that store the index as `user:://data/tntsearch/grav.index`, multiple indexes are created per language configured in Grav. For example if you have set the supported languages to `['en', 'fr', 'de']`, then when you perform an index, you will get three files: `en.index`, `fr.index`, and `de.index`. When querying the appropriate **active language** determines which index is queried. For example, performing the search on a page called `/fr/search` will result in the `fr.index` database to be used, and French results to be returned. + +Note Indexing will take longer depending on the number of languages you support as TNTSearch has to index each page in each language. + +> NOTE: While accented characters is supported in this release, there is currently no support in the underlying TNTSearch library to match non-accented characters to accented ones, so exact matches are required. + +#### Scheduler Support + +One of the great new features of Grav 1.6 is the built in **Scheduler** that allows plugin-provided functionality to be run periodically. TNTSearch is a great use-case for this capability as it allows an indexing job to be scheduled to be run every few hours without the need to manually keep things in sync. There are a few options that allow you to configure this capability. + +First note, that this scheduler functionality is disable by default, so you first have to enable the scheduler functionality in the TNTSearch plugin settings. After that you can configure how often the indexing job should run. The default is every 3 hours. Lastly, you can configure where any indexing output is logged to. + +#### Admin Page CrUD Events + +Once you have an index, TNTSearch makes use of admin events to **C**reate, **U**pdate, and **D**elete index entries when you edit pages. If your index ever looks like it's out of sync, you can simply reindex your whole site. + +#### Customizing the Search Index + +##### Adding Custom Fields + +By default the TNTSearch plugin will index the `title` and `content` of your page. This usually suffices for most cases, but there are situations where you might want to index more fields. The plugin provides an example of this by listening to the `onTNTSearchIndex` event: + +```php +public static function getSubscribedEvents() +{ + return [ + 'onTNTSearchIndex' => ['onTNTSearchIndex', 0] + ]; +} + +public function onTNTSearchIndex(Event $e) +{ + $fields = $e['fields']; + $page = $e['page']; + + if (isset($page->header()->author)) { + $fields->author = $page->header()->author; + } +} +``` + +This allows you to add an author to the indexed fields if it is set in the page frontmatter. You can add your own custom fields with a very simple plugin that listens to this event. + +##### Providing Custom Render Templates + +The TNTSearch plugin generally uses the rendered content to index with. However, there are situations where your page is actually a modular page, or built from other pages where there is no actual content on the page, or the content is not representative of the page as a whole. To combat this situation you can provide custom templates in your theme that TNTSearch can use to render the content before indexing. + +For example, say we have a homepage that is built from a few modular sub-pages with a little content below it, it's called `home.md`, so uses a `home.html.twig` file in your theme's `templates/` folder. You can create a simplified version of this template and save it as `templates/tntsearch/home.html.twig`. For this example this template looks like this: + +```twig +{% for module in page.collection() %} +
+ {{ module.content|raw }} +
+{% endfor %} + +{{ page.content|raw }} +``` + +As you can see this simply ensures the module pages as defined in the page's collection are displayed, then the actual page content is displayed. + +To instruct TNTSearch to index with this template rather than just using the Page content by itself, you just need to add an entry in the `home.md` frontmatter: + +```yaml +tntsearch: + template: 'tntsearch/home' +``` + +### Searching + +TNTSearch plugin for Grav comes with a built-in query page that is accessible via the `/search` route by default. This search page is a simple input field that will perform an Ajax query **as-you-type**. Because TNTSearch is so fast, you get a real-time search response in a similar fashion to a Google search. Also the results are returned already highlighted for matching terms. + +You can also test searching with the CLI: + +```json +$ bin/plugin tntsearch query ipsum + +{ + "number_of_hits": 3, + "execution_time": "2.101 ms", + "hits": [ + { + "link": "\/blog\/classic-modern-architecture", + "title": "Classic Modern Architecture", + "content": "...sed a odio. Curabitur ut lectus tortor. Sed ipsum<\/em> eros, egestas ut eleifend non, elementum vitae eros. Mauris felis diam, pellentesque vel lacinia ac, dictum a nunc.\nLorem ipsum<\/em> dolor sit amet, consectetur adipiscing elit. Donec ultricies tristique nulla et mattis. Phasellus id massa eget..." + }, + { + "link": "\/blog\/focus-and-blur", + "title": "Focus and Blur", + "content": "...sed a odio. Curabitur ut lectus tortor. Sed ipsum<\/em> eros, egestas ut eleifend non, elementum vitae eros. Mauris felis diam, pellentesque vel lacinia ac, dictum a nunc.\nLorem ipsum<\/em> dolor sit amet, consectetur adipiscing elit. Donec ultricies tristique nulla et mattis. Phasellus id massa eget..." + }, + { + "link": "\/blog\/london-industry", + "title": "London Industry at Night", + "content": "...sed a odio. Curabitur ut lectus tortor. Sed ipsum<\/em> eros, egestas ut eleifend non, elementum vitae eros. Mauris felis diam, pellentesque vel lacinia ac, dictum a nunc.\nLorem ipsum<\/em> dolor sit amet, consectetur adipiscing elit. Donec ultricies tristique nulla et mattis. Phasellus id massa eget..." + } + ] +} +``` + +### Customizing the Search Page + +If a physical Grav page is found for the `/search` route, TNTSearch will use that rather than the page provided by the plugin. This allows you to easily add content to your search page as you need. +If you wish to customize the actual HTML output, simply copy the `templates/search.html.twig` from the plugin to your theme and customize it. + +The actual input field can also be modified as needed by copy and editing the `templates/partials/tntsearch.html.twig` file to your theme and modify it. + +### Customizing Query Data + +By default the TNTSearch plugin for Grav, the response JSON is sent with the following structure: + +```json +{ + "number_of_hits": 3, + "execution_time": "1.000 ms", + "hits": [ + { + "link": "/page-a", + "title": "Title A", + "content": "highlighted-summary" + }, + { + "link": "/page-b", + "title": "Title B", + "content": "highlighted-summary" + }, + { + "link": "/page-c", + "title": "Title C", + "content": "highlighted-summary" + } + ] +} +``` + +There are instances where this output is not desirable or needs to be changed. TNTSearch actually provides a plugin event to allow you to manipulate this format. An example of this can be seen below: + +```php +public static function getSubscribedEvents() { + return [ + 'onTNTSearchQuery' => ['onTNTSearchQuery', 1000], + ]; +} + +public function onTNTSearchQuery(Event $e) +{ + $query = $this->grav['uri']->param('q'); + + if ($query) { + $page = $e['page']; + $query = $e['query']; + $options = $e['options']; + $fields = $e['fields']; + + $fields->results[] = $page->route(); + $e->stopPropagation(); + } +} +``` + +The important things to note are the `1000` order-value to ensure this event runs before the default event in the `tntsearch.php` plugin file. The actual event method simply sets a result array on fields to with a route, resulting in: + +```json +{ + "number_of_hits": 3, + "execution_time": "1.000 ms", + "results": ['/page-a', '/page-b', '/page-c'] +} +``` + +### Dropdown Search Field + +TNTSearch plugin can also be used to render the search as a drop-down rather than in a standard page. To do this you need to `embed` the search partial and override it to fit your needs. You could simply add this to your theme wherever you want to have an Ajax drop-down search box: + +```twig +{% embed 'partials/tntsearch.html.twig' with { limit: 10, snippet: 150, min: 3, search_type: 'auto', dropdown: true } %}{% endembed %} +``` + +Here we embed the default partial, but override the `options` by passing them in the `with` statement. It is important to notice that the `dropdown: true` is required to be set in order to be interpreted as dropdown. + +## Credits + +This plugin would not of been possible without the amazing [TNTSearch library](https://github.com/teamtnt/tntsearch) for PHP. Make sure you **star** that project on GitHub. diff --git a/plugins/tntsearch/app/history.js b/plugins/tntsearch/app/history.js new file mode 100644 index 0000000..ee3abb7 --- /dev/null +++ b/plugins/tntsearch/app/history.js @@ -0,0 +1,3 @@ +import { createBrowserHistory } from 'history'; +const history = createBrowserHistory(); +export default history; diff --git a/plugins/tntsearch/app/main.js b/plugins/tntsearch/app/main.js new file mode 100644 index 0000000..dda0af4 --- /dev/null +++ b/plugins/tntsearch/app/main.js @@ -0,0 +1,61 @@ +// polyfills +import 'babel-polyfill'; + +import domready from 'domready'; +import search from './search'; + +const GravTNTSearch = () => { + /* const uri = new URI(global.location.href, true); + history.replace({ + search: global.location.search, + hash: global.location.hash, + state: { + historyValue: uri.query.q || '', + type: 'tntsearch', + }, + });*/ + + const searchForms = document.querySelectorAll('form.tntsearch-form'); + [...searchForms].forEach((form) => { + const input = form.querySelector('.tntsearch-field'); + const clear = form.querySelector('.tntsearch-clear'); + const results = form.querySelector('.tntsearch-results'); + if (!input || !results) { return false; } + + form.addEventListener('submit', (event) => event.preventDefault()); + input.addEventListener('focus', () => search(input, results)); + input.addEventListener('input', () => { + if (clear) { + clear.style.display = ''; + } + search.cancel(); + search({ input, results }); + }); + + if (clear) { + clear.addEventListener('click', () => { + if (clear) { + clear.style.display = 'none'; + } + input.value = ''; + search.cancel(); + search({ input, results }); + }); + } + + return this; + }); + + document.addEventListener('click', (event) => { + [...searchForms].forEach((form) => { + if (!form.querySelector('.tntsearch-dropdown')) { return; } + if (!form.contains(event.target)) { + form.querySelector('.tntsearch-results').style.display = 'none'; + } + }); + }); +}; + +domready(GravTNTSearch); + +window.GravTNTSearch = GravTNTSearch; diff --git a/plugins/tntsearch/app/search.js b/plugins/tntsearch/app/search.js new file mode 100644 index 0000000..ecd6a01 --- /dev/null +++ b/plugins/tntsearch/app/search.js @@ -0,0 +1,107 @@ +import throttle from 'lodash/throttle'; +import URI from 'url-parse'; +import qs from 'querystringify'; +import history from './history'; + +export const DEFAULTS = { + uri: '', + limit: 20, + snippet: 300, + min: 3, + search_type: 'auto', + in_page: false, + live_update: true, +}; + +const historyPush = ({ value = false, params = false } = {}) => { + const uri = new URI(global.location.href, true); + + if (params === false) { + delete uri.query.q; + } else { + uri.query.q = params; + } + + const querystring = qs.stringify(uri.query, '?'); + + history.push(`${uri.pathname}${querystring}`, { + historyValue: value, type: 'tntsearch', + }); +}; + +const throttling = throttle(async ({ input, results, historyValue = false } = {}) => { + if (!input || !results) { return false; } + + const value = historyValue || input.value.trim(); + const clear = input.nextElementSibling; + const data = Object.assign({}, DEFAULTS, JSON.parse(input.dataset.tntsearch || '{}')); + + if (!value) { + results.style.display = 'none'; + + if (data.in_page) { + clear.style.display = 'none'; + + if (historyValue === false && data.live_update) { + historyPush({ value }); + } + } + + return false; + } + + if (value.length < data.min) { + return false; + } + + if (data.in_page) { + clear.style.display = ''; + } + + const params = { + q: encodeURIComponent(value), + l: data.limit, + sl: data.snippet, + search_type: data.search_type, + ajax: true, + }; + + const startEvent = new Event('tntsearch:start'); + const query = Object.keys(params) + .map(k => `${k}=${params[k]}`) + .join('&'); + + input.dispatchEvent(startEvent); + fetch(`${data.uri}?${query}`, { credentials: 'same-origin' }) + .then((response) => response.text()) + .then((response) => { + if (data.in_page && data.live_update && !historyValue) { + historyPush({ value, params: params.q }); + } + return response; + }) + .then((response) => { + const doneEvent = new Event('tntsearch:done'); + results.style.display = ''; + results.innerHTML = response; + input.dispatchEvent(doneEvent); + + return response; + }); + + return this; +}, 350, { leading: false }); + +history.listen((location) => { + if (location.state && location.state.type === 'tntsearch') { + location.state.input = document.querySelector('.tntsearch-field-inpage'); + location.state.results = document.querySelector('.tntsearch-results-inpage'); + + if (location.state.input && location.state.results) { + location.state.input.value = location.state.historyValue; + throttling({ ...location.state }); + } + } +}); + +export default throttling; diff --git a/plugins/tntsearch/assets/admin/tntsearch.css b/plugins/tntsearch/assets/admin/tntsearch.css new file mode 100644 index 0000000..37d2444 --- /dev/null +++ b/plugins/tntsearch/assets/admin/tntsearch.css @@ -0,0 +1,34 @@ +.index-status { + border: 1px solid transparent; +} +.index-status span { + padding: 0.3rem 0.7rem; + border-radius: 4px; + line-height: 1.7; + vertical-align: middle; + display: inline-block; +} +.index-status .error { + background: #ddd; + color: #c00; +} +.index-status .success { + border: 1px solid #ddd; + color: #999; +} +#admin-main .admin-block .index-status .button.critical { + background: #c00; + color: #fff; +} + +#admin-main .admin-block .index-status .button.reindex { + background: #0079BA; + color: #fff; +} + +.tntsearch-error-details { + padding: .2rem .5rem; + margin: 1rem 0; + border-radius: 3px; + display: none; +} \ No newline at end of file diff --git a/plugins/tntsearch/assets/admin/tntsearch.js b/plugins/tntsearch/assets/admin/tntsearch.js new file mode 100644 index 0000000..a7eb556 --- /dev/null +++ b/plugins/tntsearch/assets/admin/tntsearch.js @@ -0,0 +1,78 @@ +((function($) { + $(document).ready(function() { + var Request, Toastr = null; + if (typeof Grav !== 'undefined' && Grav && Grav.default && Grav.default.Utils) { + Request = Grav.default.Utils.request; + Toastr = Grav.default.Utils.toastr; + } + var indexer = $('#tntsearch-index, #admin-nav-quick-tray .tntsearch-reindex'), + current = null, currentTray = null; + if (!indexer.length) { return; } + + indexer.on('click', function(e) { + e.preventDefault(); + var target = $(e.target), + isTray = target.closest('#admin-nav-quick-tray').length, + status = indexer.siblings('.tntsearch-status'), + errorDetails = indexer.siblings('.tntsearch-error-details'); + current = status.clone(true); + + console.log(isTray); + if (isTray) { + target = target.is('i') ? target.parent() : target; + currentTray = target.find('i').attr('class'); + target.find('i').attr('class', 'fa fa-fw fa-circle-o-notch fa-spin'); + } + + errorDetails + .hide() + .empty(); + + status + .removeClass('error success') + .empty() + .html(''); + + $.ajax({ + type: 'POST', + url: GravAdmin.config.base_url_relative + '.json/task' + GravAdmin.config.param_sep + 'reindexTNTSearch', + data: { 'admin-nonce': GravAdmin.config.admin_nonce } + }).done(function(done) { + if (done.status === 'success') { + indexer.removeClass('critical').addClass('reindex'); + status.removeClass('error').addClass('success'); + Toastr.success(done.message); + } else { + indexer.removeClass('reindex').addClass('critical'); + status.removeClass('success').addClass('error'); + var error = done.message; + if (done.details) { + error += '+ {% if config.get('plugins.tntsearch.display_hits') %} + {{ "PLUGIN_TNTSEARCH.FOUND_RESULTS"|t(tntsearch_results.number_of_hits)|raw }} + {% endif %} + {% if config.get('plugins.tntsearch.display_time') %} + {{ "PLUGIN_TNTSEARCH.FOUND_IN"|t(tntsearch_results.execution_time)|raw }} + {% endif %} +
+ {% for key, val in tntsearch_results.hits %} +{{ val.content|raw }}
+ {% endfor %} ++ + + +
+ +--- +## Demo + +* [TV Shows Search](http://tntsearch.tntstudio.us/) +* [PHPUnit Documentation Search](http://phpunit.tntstudio.us) +* [City Search with n-grams](http://cities.tnt.studio/) + +## Tutorials + +* [Solving the search problem with Laravel and TNTSearch](https://tnt.studio/solving-the-search-problem-with-laravel-and-tntsearch) +* [Searching for Users with Laravel Scout and TNTSearch](https://tnt.studio/searching-for-users-with-laravel-scout-and-tntsearch) + +## Premium products + +If you're using TNT Search and finding it useful, take a look at our premium analytics tool: + + +[](https://analytics.tnt.studio) + +## Support us on Open Collective + +- [TNTSearch](https://opencollective.com/tntsearch) + +## Installation + +The easiest way to install TNTSearch is via [composer](http://getcomposer.org/): + +``` +composer require teamtnt/tntsearch +``` + +## Requirements + +Before you proceed, make sure your server meets the following requirements: + +* PHP >= 7.1 +* PDO PHP Extension +* SQLite PHP Extension +* mbstring PHP Extension + +## Examples + +### Creating an index + +In order to be able to make full text search queries, you have to create an index. + +Usage: +```php +use TeamTNT\TNTSearch\TNTSearch; + +$tnt = new TNTSearch; + +$tnt->loadConfig([ + 'driver' => 'mysql', + 'host' => 'localhost', + 'database' => 'dbname', + 'username' => 'user', + 'password' => 'pass', + 'storage' => '/var/www/tntsearch/examples/', + 'stemmer' => \TeamTNT\TNTSearch\Stemmer\PorterStemmer::class//optional +]); + +$indexer = $tnt->createIndex('name.index'); +$indexer->query('SELECT id, article FROM articles;'); +//$indexer->setLanguage('german'); +$indexer->run(); + +``` + +Important: "storage" settings marks the folder where all of your indexes +will be saved so make sure to have permission to write to this folder otherwise +you might expect the following exception thrown: + +* [PDOException] SQLSTATE[HY000] [14] unable to open database file * + +Note: If your primary key is different than `id` set it like: + +```php +$indexer->setPrimaryKey('article_id'); +``` + +### Making the primary key searchable + +By default, the primary key isn't searchable. If you want to make it searchable, simply run: + + +```php +$indexer->includePrimaryKey(); +``` + +### Searching + +Searching for a phrase or keyword is trivial: + +```php +use TeamTNT\TNTSearch\TNTSearch; + +$tnt = new TNTSearch; + +$tnt->loadConfig($config); +$tnt->selectIndex("name.index"); + +$res = $tnt->search("This is a test search", 12); + +print_r($res); //returns an array of 12 document ids that best match your query + +// to display the results you need an additional query against your application database +// SELECT * FROM articles WHERE id IN $res ORDER BY FIELD(id, $res); +``` + +The ORDER BY FIELD clause is important, otherwise the database engine will not return +the results in the required order. + +### Boolean Search + +```php +use TeamTNT\TNTSearch\TNTSearch; + +$tnt = new TNTSearch; + +$tnt->loadConfig($config); +$tnt->selectIndex("name.index"); + +//this will return all documents that have romeo in it but not juliet +$res = $tnt->searchBoolean("romeo -juliet"); + +//returns all documents that have romeo or hamlet in it +$res = $tnt->searchBoolean("romeo or hamlet"); + +//returns all documents that have either romeo AND juliet or prince AND hamlet +$res = $tnt->searchBoolean("(romeo juliet) or (prince hamlet)"); + +``` + +### Fuzzy Search + +The fuzziness can be tweaked by setting the following member variables: + +```php +public $fuzzy_prefix_length = 2; +public $fuzzy_max_expansions = 50; +public $fuzzy_distance = 2; //represents the Levenshtein distance; +``` + +```php +use TeamTNT\TNTSearch\TNTSearch; + +$tnt = new TNTSearch; + +$tnt->loadConfig($config); +$tnt->selectIndex("name.index"); +$tnt->fuzziness = true; + +//when the fuzziness flag is set to true, the keyword juleit will return +//documents that match the word juliet, the default Levenshtein distance is 2 +$res = $tnt->search("juleit"); + +``` +## Updating the index + +Once you created an index, you don't need to reindex it each time you make some changes +to your document collection. TNTSearch supports dynamic index updates. + +```php +use TeamTNT\TNTSearch\TNTSearch; + +$tnt = new TNTSearch; + +$tnt->loadConfig($config); +$tnt->selectIndex("name.index"); + +$index = $tnt->getIndex(); + +//to insert a new document to the index +$index->insert(['id' => '11', 'title' => 'new title', 'article' => 'new article']); + +//to update an existing document +$index->update(11, ['id' => '11', 'title' => 'updated title', 'article' => 'updated article']); + +//to delete the document from index +$index->delete(12); +``` + +## Custom Tokenizer +First, create your own Tokenizer class. It should extend AbstractTokenizer class, define +word split $pattern value and must implement TokenizerInterface: + +``` php + +use TeamTNT\TNTSearch\Support\AbstractTokenizer; +use TeamTNT\TNTSearch\Support\TokenizerInterface; + +class SomeTokenizer extends AbstractTokenizer implements TokenizerInterface +{ + static protected $pattern = '/[\s,\.]+/'; + + public function tokenize($text) { + return preg_split($this->getPattern(), strtolower($text), -1, PREG_SPLIT_NO_EMPTY); + } +} +``` + +This tokenizer will split words using spaces, commas and periods. + +After you have the tokenizer ready, you should pass it to `TNTIndexer` via `setTokenizer` method. + +``` php +$someTokenizer = new SomeTokenizer; + +$indexer = new TNTIndexer; +$indexer->setTokenizer($someTokenizer); +``` + +Another way would be to pass the tokenizer via config: + +```php +use TeamTNT\TNTSearch\TNTSearch; + +$tnt = new TNTSearch; + +$tnt->loadConfig([ + 'driver' => 'mysql', + 'host' => 'localhost', + 'database' => 'dbname', + 'username' => 'user', + 'password' => 'pass', + 'storage' => '/var/www/tntsearch/examples/', + 'stemmer' => \TeamTNT\TNTSearch\Stemmer\PorterStemmer::class//optional, + 'tokenizer' => \TeamTNT\TNTSearch\Support\SomeTokenizer::class +]); + +$indexer = $tnt->createIndex('name.index'); +$indexer->query('SELECT id, article FROM articles;'); +$indexer->run(); + +``` + +## Geo Search + +### Indexing + +```php +$candyShopIndexer = new TNTGeoIndexer; +$candyShopIndexer->loadConfig($config); +$candyShopIndexer->createIndex('candyShops.index'); +$candyShopIndexer->query('SELECT id, longitude, latitude FROM candy_shops;'); +$candyShopIndexer->run(); +``` +### Searching + +```php +$currentLocation = [ + 'longitude' => 11.576124, + 'latitude' => 48.137154 +]; + +$distance = 2; //km + +$candyShopIndex = new TNTGeoSearch(); +$candyShopIndex->loadConfig($config); +$candyShopIndex->selectIndex('candyShops.index'); + +$candyShops = $candyShopIndex->findNearest($currentLocation, $distance, 10); +``` + +## Classification + +```php +use TeamTNT\TNTSearch\Classifier\TNTClassifier; + +$classifier = new TNTClassifier(); +$classifier->learn("A great game", "Sports"); +$classifier->learn("The election was over", "Not sports"); +$classifier->learn("Very clean match", "Sports"); +$classifier->learn("A clean but forgettable game", "Sports"); + +$guess = $classifier->predict("It was a close election"); +var_dump($guess['label']); //returns "Not sports" + +``` + +### Saving the classifier + +```php +$classifier->save('sports.cls'); +``` + +### Loading the classifier + +```php +$classifier = new TNTClassifier(); +$classifier->load('sports.cls'); +``` + +## Drivers + +* [TNTSearch Driver for Laravel Scout](https://github.com/teamtnt/laravel-scout-tntsearch-driver) + +## PS4Ware + +You're free to use this package, but if it makes it to your production environment, we would highly appreciate you sending us a PS4 game of your choice. This way you support us to further develop and add new features. + +Our address is: TNT Studio, Sv. Mateja 19, 10010 Zagreb, Croatia. + +We'll publish all received games [here][link-ps4ware] + +[link-ps4ware]: https://github.com/teamtnt/tntsearch/blob/master/PS4Ware.md + +## Support [![OpenCollective](https://opencollective.com/tntsearch/backers/badge.svg)](#backers) [![OpenCollective](https://opencollective.com/tntsearch/sponsors/badge.svg)](#sponsors) + + + +### Backers + +Support us with a monthly donation and help us continue our activities. [[Become a backer](https://opencollective.com/tntsearch#backer)] + +## Sponsors + +Become a sponsor and get your logo on our README on Github with a link to your site. [[Become a sponsor](https://opencollective.com/tntsearch#sponsor)] + +## Credits + +- [Nenad Tičarić][link-author] +- [All Contributors][link-contributors] + +## License + +The MIT License (MIT). Please see [License File](LICENSE.md) for more information. + +[ico-version]: https://img.shields.io/packagist/v/teamtnt/tntsearch.svg?style=flat-square +[ico-license]: https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat-square +[ico-downloads]: https://img.shields.io/packagist/dt/teamtnt/tntsearch.svg?style=flat-square + +[link-packagist]: https://packagist.org/packages/teamtnt/tntsearch +[link-downloads]: https://packagist.org/packages/teamtnt/tntsearch +[link-author]: https://github.com/nticaric +[link-contributors]: ../../contributors + +--- +From Croatia with ♥ by TNT Studio ([@tntstudiohr](https://twitter.com/tntstudiohr), [blog](https://tnt.studio)) diff --git a/plugins/tntsearch/vendor/teamtnt/tntsearch/composer.json b/plugins/tntsearch/vendor/teamtnt/tntsearch/composer.json new file mode 100644 index 0000000..f1a0bad --- /dev/null +++ b/plugins/tntsearch/vendor/teamtnt/tntsearch/composer.json @@ -0,0 +1,42 @@ +{ + "name": "teamtnt/tntsearch", + "type": "library", + "description": "A fully featured full text search engine written in PHP", + "keywords": [ + "teamtnt", + "tntsearch", + "search", + "fulltext", + "geosearch", + "text classification", + "bm25", + "stemming", + "fuzzy search" + ], + "homepage": "https://github.com/teamtnt/tntsearch", + "license": "MIT", + "authors": [{ + "name": "Nenad Tičarić", + "email": "nticaric@gmail.com", + "homepage": "http://www.tntstudio.us", + "role": "Developer" + }], + "require": { + "php": "~7.1|^8", + "ext-pdo_sqlite": "*", + "ext-sqlite3": "*", + "ext-mbstring": "*" + }, + "require-dev": { + "phpunit/phpunit": "7.*|8.*|9.*", + "symfony/var-dumper": "^4|^5.2" + }, + "autoload": { + "psr-4": { + "TeamTNT\\TNTSearch\\": "src" + }, + "files": [ + "helper/helpers.php" + ] + } +} \ No newline at end of file diff --git a/plugins/tntsearch/vendor/teamtnt/tntsearch/helper/helpers.php b/plugins/tntsearch/vendor/teamtnt/tntsearch/helper/helpers.php new file mode 100644 index 0000000..e0a663b --- /dev/null +++ b/plugins/tntsearch/vendor/teamtnt/tntsearch/helper/helpers.php @@ -0,0 +1,25 @@ += 0 && strpos($haystack, $needle, $temp) !== false); + } +} + +if (!function_exists('fuzzyMatch')) { + function fuzzyMatch($pattern, $items) + { + $fm = new TeamTNT\TNTSearch\TNTFuzzyMatch; + return $fm->fuzzyMatch($pattern, $items); + } +} + +if (!function_exists('fuzzyMatchFromFile')) { + function fuzzyMatchFromFile($pattern, $path) + { + $fm = new TeamTNT\TNTSearch\TNTFuzzyMatch; + return $fm->fuzzyMatchFromFile($pattern, $path); + } +} diff --git a/plugins/tntsearch/vendor/teamtnt/tntsearch/phpunit.php b/plugins/tntsearch/vendor/teamtnt/tntsearch/phpunit.php new file mode 100644 index 0000000..22134c3 --- /dev/null +++ b/plugins/tntsearch/vendor/teamtnt/tntsearch/phpunit.php @@ -0,0 +1,15 @@ + +