lots of changes for the new unified search

src-ui/src/app/app-routing.module.ts

@@ -10,7 +10,6 @@ import { LogsComponent } from './components/manage/logs/logs.component';
 import { SettingsComponent } from './components/manage/settings/settings.component';
 import { TagListComponent } from './components/manage/tag-list/tag-list.component';
 import { NotFoundComponent } from './components/not-found/not-found.component';
-import { SearchComponent } from './components/search/search.component';
 
 const routes: Routes = [
   {path: '', redirectTo: 'dashboard', pathMatch: 'full'},
@@ -18,7 +17,6 @@ const routes: Routes = [
     {path: 'dashboard', component: DashboardComponent },
     {path: 'documents', component: DocumentListComponent },
     {path: 'view/:id', component: DocumentListComponent },
-    {path: 'search', component: SearchComponent },
     {path: 'documents/:id', component: DocumentDetailComponent },
   
     {path: 'tags', component: TagListComponent },

src-ui/src/app/app.module.ts

@@ -21,8 +21,6 @@ import { CorrespondentEditDialogComponent } from './components/manage/correspond
 import { TagEditDialogComponent } from './components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component';
 import { DocumentTypeEditDialogComponent } from './components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component';
 import { TagComponent } from './components/common/tag/tag.component';
-import { SearchComponent } from './components/search/search.component';
-import { ResultHighlightComponent } from './components/search/result-highlight/result-highlight.component';
 import { PageHeaderComponent } from './components/common/page-header/page-header.component';
 import { AppFrameComponent } from './components/app-frame/app-frame.component';
 import { ToastsComponent } from './components/common/toasts/toasts.component';
@@ -104,8 +102,6 @@ registerLocaleData(localeEs)
     TagEditDialogComponent,
     DocumentTypeEditDialogComponent,
     TagComponent,
-    SearchComponent,
-    ResultHighlightComponent,
     PageHeaderComponent,
     AppFrameComponent,
     ToastsComponent,

src-ui/src/app/components/app-frame/app-frame.component.ts

@@ -10,6 +10,8 @@ import { SearchService } from 'src/app/services/rest/search.service';
 import { environment } from 'src/environments/environment';
 import { DocumentDetailComponent } from '../document-detail/document-detail.component';
 import { Meta } from '@angular/platform-browser';
+import { DocumentListViewService } from 'src/app/services/document-list-view.service';
+import { FILTER_FULLTEXT_QUERY } from 'src/app/data/filter-rule-type';
 
 @Component({
   selector: 'app-app-frame',
@@ -24,6 +26,7 @@ export class AppFrameComponent implements OnInit {
     private openDocumentsService: OpenDocumentsService,
     private searchService: SearchService,
     public savedViewService: SavedViewService,
+    private list: DocumentListViewService,
     private meta: Meta
     ) {
 
@@ -74,7 +77,7 @@ export class AppFrameComponent implements OnInit {
 
   search() {
     this.closeMenu()
-    this.router.navigate(['search'], {queryParams: {query: this.searchField.value}})
+    this.list.quickFilter([{rule_type: FILTER_FULLTEXT_QUERY, value: this.searchField.value}])
   }
 
   closeDocument(d: PaperlessDocument) {

src-ui/src/app/components/document-detail/document-detail.component.ts

@@ -20,6 +20,7 @@ import { ToastService } from 'src/app/services/toast.service';
 import { TextComponent } from '../common/input/text/text.component';
 import { SettingsService, SETTINGS_KEYS } from 'src/app/services/settings.service';
 import { PaperlessDocumentSuggestions } from 'src/app/data/paperless-document-suggestions';
+import { FILTER_FULLTEXT_MORELIKE } from 'src/app/data/filter-rule-type';
 
 @Component({
   selector: 'app-document-detail',
@@ -219,7 +220,7 @@ export class DocumentDetailComponent implements OnInit {
   }
 
   moreLike() {
-    this.router.navigate(["search"], {queryParams: {more_like:this.document.id}})
+    this.documentListViewService.quickFilter([{rule_type: FILTER_FULLTEXT_MORELIKE, value: this.documentId.toString()}])
   }
 
   hasNext() {

src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html

@@ -25,14 +25,14 @@
           </h5>
         </div>
         <p class="card-text">
-          <app-result-highlight *ngIf="getDetailsAsHighlight()" class="result-content" [highlights]="getDetailsAsHighlight()"></app-result-highlight>
+          <span *ngIf="document.__search_hit__" [innerHtml]="document.__search_hit__.highlights"></span>
-          <span *ngIf="getDetailsAsString()" class="result-content">{{getDetailsAsString()}}</span>
+          <span *ngIf="!document.__search_hit__" class="result-content">{{contentTrimmed}}</span>
         </p>
 
 
         <div class="d-flex flex-column flex-md-row align-items-md-center">
           <div class="btn-group">
-            <a routerLink="/search" [queryParams]="{'more_like': document.id}" class="btn btn-sm btn-outline-secondary" *ngIf="moreLikeThis">
+            <a class="btn btn-sm btn-outline-secondary" (click)="clickMoreLike.emit()">
               <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-three-dots" viewBox="0 0 16 16">
                 <path fill-rule="evenodd" d="M3 9.5a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3zm5 0a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3zm5 0a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3z"/>
               </svg>&nbsp;<span class="d-block d-md-inline" i18n>More like this</span>
@@ -62,9 +62,9 @@
           </div>
 
           <div class="list-group list-group-horizontal border-0 card-info ml-md-auto mt-2 mt-md-0">
-            <div *ngIf="searchScore" class="list-group-item bg-light text-dark p-1 mr-5 border-0 d-flex search-score">
+            <div *ngIf="document.__search_hit__" class="list-group-item bg-light text-dark p-1 mr-5 border-0 d-flex search-score">
               <small class="text-muted" i18n>Score:</small>
-              <ngb-progressbar [type]="searchScoreClass" [value]="searchScore" class="search-score-bar mx-2 mt-1" [max]="1"></ngb-progressbar>
+              <ngb-progressbar [type]="searchScoreClass" [value]="document.__search_hit__.score" class="search-score-bar mx-2 mt-1" [max]="1"></ngb-progressbar>
             </div>
             <button *ngIf="document.document_type" type="button" class="list-group-item btn btn-sm bg-light text-dark p-1 border-0 mr-2" title="Filter by document type"
              (click)="clickDocumentType.emit(document.document_type);$event.stopPropagation()">

src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss

@@ -60,3 +60,8 @@
     padding-top: 0.35rem !important;
   }
 }
+
+span ::ng-deep .match {
+  color: black;
+  background-color: rgb(255, 211, 66);
+}

src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts

@@ -4,6 +4,8 @@ import { PaperlessDocument } from 'src/app/data/paperless-document';
 import { DocumentService } from 'src/app/services/rest/document.service';
 import { SettingsService, SETTINGS_KEYS } from 'src/app/services/settings.service';
 import { NgbPopover } from '@ng-bootstrap/ng-bootstrap';
+import { DocumentListViewService } from 'src/app/services/document-list-view.service';
+import { FILTER_FULLTEXT_MORELIKE } from 'src/app/data/filter-rule-type';
 
 @Component({
   selector: 'app-document-card-large',
@@ -24,15 +26,9 @@ export class DocumentCardLargeComponent implements OnInit {
     return this.toggleSelected.observers.length > 0
   }
 
-  @Input()
-  moreLikeThis: boolean = false
-
   @Input()
   document: PaperlessDocument
 
-  @Input()
-  details: any
-
   @Output()
   clickTag = new EventEmitter<number>()
 
@@ -42,6 +38,9 @@ export class DocumentCardLargeComponent implements OnInit {
   @Output()
   clickDocumentType = new EventEmitter<number>()
 
+  @Output()
+  clickMoreLike= new EventEmitter()
+
   @Input()
   searchScore: number
 
@@ -67,19 +66,6 @@ export class DocumentCardLargeComponent implements OnInit {
     return this.settingsService.get(SETTINGS_KEYS.DARK_MODE_THUMB_INVERTED)
   }
 
-  getDetailsAsString() {
-    if (typeof this.details === 'string') {
-      return this.details.substring(0, 500)
-    }
-  }
-
-  getDetailsAsHighlight() {
-    //TODO: this is not an exact typecheck, can we do better
-    if (this.details instanceof Array) {
-      return this.details
-    }
-  }
-
   getThumbUrl() {
     return this.documentService.getThumbUrl(this.document.id)
   }
@@ -116,4 +102,8 @@ export class DocumentCardLargeComponent implements OnInit {
   mouseLeaveCard() {
     this.popover.close()
   }
+
+  get contentTrimmed() {
+    return this.document.content.substr(0, 500)
+  }
 }

src-ui/src/app/components/document-list/document-list.component.html

@@ -90,7 +90,7 @@
 </div>
 
 <div *ngIf="displayMode == 'largeCards'">
-  <app-document-card-large [selected]="list.isSelected(d)" (toggleSelected)="toggleSelected(d, $event)" *ngFor="let d of list.documents; trackBy: trackByDocumentId" [document]="d" [details]="d.content" (clickTag)="clickTag($event)" (clickCorrespondent)="clickCorrespondent($event)" (clickDocumentType)="clickDocumentType($event)">
+  <app-document-card-large [selected]="list.isSelected(d)" (toggleSelected)="toggleSelected(d, $event)" *ngFor="let d of list.documents; trackBy: trackByDocumentId" [document]="d" (clickTag)="clickTag($event)" (clickCorrespondent)="clickCorrespondent($event)" (clickDocumentType)="clickDocumentType($event)" (clickMoreLike)="clickMoreLike(d.id)">
   </app-document-card-large>
 </div>
 

src-ui/src/app/components/document-list/document-list.component.ts

@@ -207,6 +207,13 @@ export class DocumentListComponent implements OnInit, OnDestroy {
     })
   }
 
+  clickMoreLike(documentID: number) {
+    this.list.selectNone()
+    setTimeout(() => {
+      //this.filterEditor.moreLikeThis(doc)
+    })
+  }
+
   trackByDocumentId(index, item: PaperlessDocument) {
     return item.id
   }

src-ui/src/app/components/document-list/filter-editor/filter-editor.component.html

@@ -1,7 +1,6 @@
 <div class="row">
    <div class="col mb-2 mb-xl-0">
      <div class="form-inline d-flex align-items-center">
-         <label class="text-muted mr-2 mb-0" i18n>Filter by:</label>
          <div class="input-group input-group-sm flex-fill w-auto">
            <div class="input-group-prepend" ngbDropdown>
             <button class="btn btn-outline-primary" ngbDropdownToggle>{{textFilterTargetName}}</button>   
@@ -9,7 +8,8 @@
               <button *ngFor="let t of textFilterTargets" ngbDropdownItem [class.active]="textFilterTarget == t.id" (click)="changeTextFilterTarget(t.id)">{{t.name}}</button>
             </div>
           </div>
-          <input class="form-control form-control-sm" type="text" [(ngModel)]="textFilter">
+          <input class="form-control form-control-sm" type="text" [(ngModel)]="textFilter" *ngIf="textFilterTarget != 'fulltext-morelike'">
+          <span class="form-control form-control-sm text-truncate" *ngIf="textFilterTarget == 'fulltext-morelike'">{{_moreLikeDoc?.title}}</span>
          </div>
      </div>
   </div>

src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts

@@ -8,13 +8,17 @@ import { DocumentTypeService } from 'src/app/services/rest/document-type.service
 import { TagService } from 'src/app/services/rest/tag.service';
 import { CorrespondentService } from 'src/app/services/rest/correspondent.service';
 import { FilterRule } from 'src/app/data/filter-rule';
-import { FILTER_ADDED_AFTER, FILTER_ADDED_BEFORE, FILTER_ASN, FILTER_CORRESPONDENT, FILTER_CREATED_AFTER, FILTER_CREATED_BEFORE, FILTER_DOCUMENT_TYPE, FILTER_HAS_ANY_TAG, FILTER_HAS_TAG, FILTER_TITLE, FILTER_TITLE_CONTENT } from 'src/app/data/filter-rule-type';
+import { FILTER_ADDED_AFTER, FILTER_ADDED_BEFORE, FILTER_ASN, FILTER_CORRESPONDENT, FILTER_CREATED_AFTER, FILTER_CREATED_BEFORE, FILTER_DOCUMENT_TYPE, FILTER_FULLTEXT_MORELIKE, FILTER_FULLTEXT_QUERY, FILTER_HAS_ANY_TAG, FILTER_HAS_TAG, FILTER_TITLE, FILTER_TITLE_CONTENT } from 'src/app/data/filter-rule-type';
 import { FilterableDropdownSelectionModel } from '../../common/filterable-dropdown/filterable-dropdown.component';
 import { ToggleableItemState } from '../../common/filterable-dropdown/toggleable-dropdown-button/toggleable-dropdown-button.component';
+import { DocumentService } from 'src/app/services/rest/document.service';
+import { PaperlessDocument } from 'src/app/data/paperless-document';
 
 const TEXT_FILTER_TARGET_TITLE = "title"
 const TEXT_FILTER_TARGET_TITLE_CONTENT = "title-content"
 const TEXT_FILTER_TARGET_ASN = "asn"
+const TEXT_FILTER_TARGET_FULLTEXT_QUERY = "fulltext-query"
+const TEXT_FILTER_TARGET_FULLTEXT_MORELIKE = "fulltext-morelike"
 
 @Component({
   selector: 'app-filter-editor',
@@ -64,7 +68,8 @@ export class FilterEditorComponent implements OnInit, OnDestroy {
   constructor(
     private documentTypeService: DocumentTypeService,
     private tagService: TagService,
-    private correspondentService: CorrespondentService
+    private correspondentService: CorrespondentService,
+    private documentService: DocumentService
   ) { }
 
   tags: PaperlessTag[] = []
@@ -72,12 +77,21 @@ export class FilterEditorComponent implements OnInit, OnDestroy {
   documentTypes: PaperlessDocumentType[] = []
 
   _textFilter = ""
+  _moreLikeId: number
+  _moreLikeDoc: PaperlessDocument
 
-  textFilterTargets = [
+  get textFilterTargets() {
+    let targets = [
       {id: TEXT_FILTER_TARGET_TITLE, name: $localize`Title`},
       {id: TEXT_FILTER_TARGET_TITLE_CONTENT, name: $localize`Title & content`},
-    {id: TEXT_FILTER_TARGET_ASN, name: $localize`ASN`}
+      {id: TEXT_FILTER_TARGET_ASN, name: $localize`ASN`},
+      {id: TEXT_FILTER_TARGET_FULLTEXT_QUERY, name: $localize`Fulltext search`}
     ]
+    if (this.textFilterTarget == TEXT_FILTER_TARGET_FULLTEXT_MORELIKE) {
+      targets.push({id: TEXT_FILTER_TARGET_FULLTEXT_MORELIKE, name: $localize`More like`})
+    }
+    return targets
+  }
 
   textFilterTarget = TEXT_FILTER_TARGET_TITLE_CONTENT
 
@@ -101,6 +115,7 @@ export class FilterEditorComponent implements OnInit, OnDestroy {
     this.tagSelectionModel.clear(false)
     this.correspondentSelectionModel.clear(false)
     this._textFilter = null
+    this._moreLikeId = null
     this.dateAddedBefore = null
     this.dateAddedAfter = null
     this.dateCreatedBefore = null
@@ -120,6 +135,17 @@ export class FilterEditorComponent implements OnInit, OnDestroy {
           this._textFilter = rule.value
           this.textFilterTarget = TEXT_FILTER_TARGET_ASN
           break
+        case FILTER_FULLTEXT_QUERY:
+          this._textFilter = rule.value
+          this.textFilterTarget = TEXT_FILTER_TARGET_FULLTEXT_QUERY
+          break
+        case FILTER_FULLTEXT_MORELIKE:
+          this._moreLikeId = +rule.value
+          this.textFilterTarget = TEXT_FILTER_TARGET_FULLTEXT_MORELIKE
+          this.documentService.get(this._moreLikeId).subscribe(result => {
+            this._moreLikeDoc = result
+          })
+          break
         case FILTER_CREATED_AFTER:
           this.dateCreatedAfter = rule.value
           break
@@ -159,6 +185,12 @@ export class FilterEditorComponent implements OnInit, OnDestroy {
     if (this._textFilter && this.textFilterTarget == TEXT_FILTER_TARGET_ASN) {
       filterRules.push({rule_type: FILTER_ASN, value: this._textFilter})
     }
+    if (this._textFilter && this.textFilterTarget == TEXT_FILTER_TARGET_FULLTEXT_QUERY) {
+      filterRules.push({rule_type: FILTER_FULLTEXT_QUERY, value: this._textFilter})
+    }
+    if (this._moreLikeId && this.textFilterTarget == TEXT_FILTER_TARGET_FULLTEXT_MORELIKE) {
+      filterRules.push({rule_type: FILTER_FULLTEXT_MORELIKE, value: this._moreLikeId?.toString()})
+    }
     if (this.tagSelectionModel.isNoneSelected()) {
       filterRules.push({rule_type: FILTER_HAS_ANY_TAG, value: "false"})
     } else {
@@ -232,6 +264,7 @@ export class FilterEditorComponent implements OnInit, OnDestroy {
   }
 
   resetSelected() {
+    this.textFilterTarget = TEXT_FILTER_TARGET_TITLE_CONTENT
     this.reset.next()
   }
 

src-ui/src/app/components/search/result-highlight/result-highlight.component.html

@@ -1,3 +0,0 @@
-... <span *ngFor="let fragment of highlights">
-    <span *ngFor="let token of fragment" [class.match]="token.highlight">{{token.text}}</span> ... 
-</span>

src-ui/src/app/components/search/result-highlight/result-highlight.component.scss

@@ -1,4 +0,0 @@
-.match {
-    color: black;
-    background-color: rgb(255, 211, 66);
-}

src-ui/src/app/components/search/result-highlight/result-highlight.component.spec.ts

@@ -1,25 +0,0 @@
-import { ComponentFixture, TestBed } from '@angular/core/testing';
-
-import { ResultHighlightComponent } from './result-highlight.component';
-
-describe('ResultHighlightComponent', () => {
-  let component: ResultHighlightComponent;
-  let fixture: ComponentFixture<ResultHighlightComponent>;
-
-  beforeEach(async () => {
-    await TestBed.configureTestingModule({
-      declarations: [ ResultHighlightComponent ]
-    })
-    .compileComponents();
-  });
-
-  beforeEach(() => {
-    fixture = TestBed.createComponent(ResultHighlightComponent);
-    component = fixture.componentInstance;
-    fixture.detectChanges();
-  });
-
-  it('should create', () => {
-    expect(component).toBeTruthy();
-  });
-});

src-ui/src/app/components/search/result-highlight/result-highlight.component.ts

@@ -1,19 +0,0 @@
-import { Component, Input, OnInit } from '@angular/core';
-import { SearchHitHighlight } from 'src/app/data/search-result';
-
-@Component({
-  selector: 'app-result-highlight',
-  templateUrl: './result-highlight.component.html',
-  styleUrls: ['./result-highlight.component.scss']
-})
-export class ResultHighlightComponent implements OnInit {
-
-  constructor() { }
-
-  @Input()
-  highlights: SearchHitHighlight[][]
-
-  ngOnInit(): void {
-  }
-
-}

src-ui/src/app/components/search/search.component.html

@@ -1,26 +0,0 @@
-<app-page-header i18n-title title="Search results">
-</app-page-header>
-
-<div *ngIf="errorMessage" class="alert alert-danger" i18n>Invalid search query: {{errorMessage}}</div>
-
-<p *ngIf="more_like" i18n>Showing documents similar to <a routerLink="/documents/{{more_like}}">{{more_like_doc?.original_file_name}}</a></p>
-
-<p *ngIf="query">
-    <ng-container i18n>Search query: <i>{{query}}</i></ng-container>
-    <ng-container *ngIf="correctedQuery">
-        - <ng-container i18n>Did you mean "<a [routerLink]="" (click)="searchCorrectedQuery()">{{correctedQuery}}</a>"?</ng-container>
-    </ng-container>
-</p>
-
-<div *ngIf="!errorMessage" [class.result-content-searching]="searching" infiniteScroll (scrolled)="onScroll()">
-    <p i18n>{resultCount, plural, =0 {No results} =1 {One result} other {{{resultCount}} results}}</p>
-    <ng-container *ngFor="let result of results">
-        <app-document-card-large *ngIf="result.document"
-            [document]="result.document"
-            [details]="result.highlights"
-            [searchScore]="result.score / maxScore"
-            [moreLikeThis]="true">
-        </app-document-card-large>
-    </ng-container>
-
-</div>

src-ui/src/app/components/search/search.component.scss

@@ -1,15 +0,0 @@
-.result-content {
-    color: darkgray;
-}
-
-.doc-img {
-    object-fit: cover;
-    object-position: top;
-    height: 100%;
-    position: absolute;
-
-}
-
-.result-content-searching {
-    opacity: 0.3;
-}

src-ui/src/app/components/search/search.component.spec.ts

@@ -1,25 +0,0 @@
-import { ComponentFixture, TestBed } from '@angular/core/testing';
-
-import { SearchComponent } from './search.component';
-
-describe('SearchComponent', () => {
-  let component: SearchComponent;
-  let fixture: ComponentFixture<SearchComponent>;
-
-  beforeEach(async () => {
-    await TestBed.configureTestingModule({
-      declarations: [ SearchComponent ]
-    })
-    .compileComponents();
-  });
-
-  beforeEach(() => {
-    fixture = TestBed.createComponent(SearchComponent);
-    component = fixture.componentInstance;
-    fixture.detectChanges();
-  });
-
-  it('should create', () => {
-    expect(component).toBeTruthy();
-  });
-});

src-ui/src/app/components/search/search.component.ts

@@ -1,95 +0,0 @@
-import { Component, OnInit } from '@angular/core';
-import { ActivatedRoute, Router } from '@angular/router';
-import { PaperlessDocument } from 'src/app/data/paperless-document';
-import { PaperlessDocumentType } from 'src/app/data/paperless-document-type';
-import { SearchHit } from 'src/app/data/search-result';
-import { DocumentService } from 'src/app/services/rest/document.service';
-import { SearchService } from 'src/app/services/rest/search.service';
-
-@Component({
-  selector: 'app-search',
-  templateUrl: './search.component.html',
-  styleUrls: ['./search.component.scss']
-})
-export class SearchComponent implements OnInit {
-
-  results: SearchHit[] = []
-
-  query: string = ""
-
-  more_like: number
-
-  more_like_doc: PaperlessDocument
-
-  searching = false
-
-  currentPage = 1
-
-  pageCount = 1
-
-  resultCount
-
-  correctedQuery: string = null
-
-  errorMessage: string
-
-  get maxScore() {
-    return this.results?.length > 0 ? this.results[0].score : 100
-  }
-
-  constructor(private searchService: SearchService, private route: ActivatedRoute, private router: Router, private documentService: DocumentService) { }
-
-  ngOnInit(): void {
-    this.route.queryParamMap.subscribe(paramMap => {
-      window.scrollTo(0, 0)
-      this.query = paramMap.get('query')
-      this.more_like = paramMap.has('more_like') ? +paramMap.get('more_like') : null
-      if (this.more_like) {
-        this.documentService.get(this.more_like).subscribe(r => {
-          this.more_like_doc = r
-        })
-      } else {
-        this.more_like_doc = null
-      }
-      this.searching = true
-      this.currentPage = 1
-      this.loadPage()
-    })
-
-  }
-
-  searchCorrectedQuery() {
-    this.router.navigate(["search"], {queryParams: {query: this.correctedQuery, more_like: this.more_like}})
-  }
-
-  loadPage(append: boolean = false) {
-    this.errorMessage = null
-    this.correctedQuery = null
-
-    this.searchService.search(this.query, this.currentPage, this.more_like).subscribe(result => {
-      if (append) {
-        this.results.push(...result.results)
-      } else {
-        this.results = result.results
-      }
-      this.pageCount = result.page_count
-      this.searching = false
-      this.resultCount = result.count
-      this.correctedQuery = result.corrected_query
-    }, error => {
-      this.searching = false
-      this.resultCount = 1
-      this.pageCount = 1
-      this.results = []
-      this.errorMessage = error.error
-    })
-  }
-
-  onScroll() {
-    if (this.currentPage < this.pageCount) {
-      this.currentPage += 1
-      this.loadPage(true)
-    }
-  }
-
-}

src-ui/src/app/data/filter-rule-type.ts

@@ -22,6 +22,9 @@ export const FILTER_ASN_ISNULL = 18
 
 export const FILTER_TITLE_CONTENT = 19
 
+export const FILTER_FULLTEXT_QUERY = 20
+export const FILTER_FULLTEXT_MORELIKE = 21
+
 export const FILTER_RULE_TYPES: FilterRuleType[] = [
 
   {id: FILTER_TITLE, filtervar: "title__icontains", datatype: "string", multi: false, default: ""},
@@ -51,7 +54,11 @@ export const FILTER_RULE_TYPES: FilterRuleType[] = [
   {id: FILTER_MODIFIED_AFTER, filtervar: "modified__date__gt", datatype: "date", multi: false},
   {id: FILTER_ASN_ISNULL, filtervar: "archive_serial_number__isnull", datatype: "boolean", multi: false},
 
-  {id: FILTER_TITLE_CONTENT, filtervar: "title_content", datatype: "string", multi: false}
+  {id: FILTER_TITLE_CONTENT, filtervar: "title_content", datatype: "string", multi: false},
+
+  {id: FILTER_FULLTEXT_QUERY, filtervar: "query", datatype: "string", multi: false},
+
+  {id: FILTER_FULLTEXT_MORELIKE, filtervar: "more_like_id", datatype: "number", multi: false},
 ]
 
 export interface FilterRuleType {

src-ui/src/app/data/paperless-document.ts

@@ -4,6 +4,15 @@ import { PaperlessTag } from './paperless-tag'
 import { PaperlessDocumentType } from './paperless-document-type'
 import { Observable } from 'rxjs'
 
+export interface SearchHit {
+
+  score?: number
+  rank?: number
+
+  highlights?: string
+
+}
+
 export interface PaperlessDocument extends ObjectWithId {
 
     correspondent$?: Observable<PaperlessCorrespondent>
@@ -40,4 +49,6 @@ export interface PaperlessDocument extends ObjectWithId {
 
     archive_serial_number?: number
 
+    __search_hit__?: SearchHit
+
 }

src-ui/src/app/data/search-result.ts

@@ -1,29 +0,0 @@
-import { PaperlessDocument } from './paperless-document'
-
-export class SearchHitHighlight {
-  text?: string
-  term?: number
-}
-
-export interface SearchHit {
-  id?: number
-  title?: string
-  score?: number
-  rank?: number
-
-  highlights?: SearchHitHighlight[][]
-  document?: PaperlessDocument
-}
-
-export interface SearchResult {
-
-  count?: number
-  page?: number
-  page_count?: number
-
-  corrected_query?: string
-
-  results?: SearchHit[]
-
-
-}

src-ui/src/app/services/document-list-view.service.ts

@@ -1,7 +1,9 @@
+import { Route } from '@angular/compiler/src/core';
 import { Injectable } from '@angular/core';
-import { Router } from '@angular/router';
+import { ActivatedRoute, Router } from '@angular/router';
 import { Observable } from 'rxjs';
 import { cloneFilterRules, FilterRule } from '../data/filter-rule';
+import { FILTER_FULLTEXT_MORELIKE, FILTER_FULLTEXT_QUERY } from '../data/filter-rule-type';
 import { PaperlessDocument } from '../data/paperless-document';
 import { PaperlessSavedView } from '../data/paperless-saved-view';
 import { DOCUMENT_LIST_SERVICE } from '../data/storage-keys';
@@ -207,8 +209,12 @@ export class DocumentListViewService {
     this.activeListViewState.currentPage = 1
     this.reduceSelectionToFilter()
     this.saveDocumentListView()
+    if (this.router.url == "/documents") {
+      this.reload()
+    } else {
       this.router.navigate(["documents"])
     }
+  }
 
   getLastPage(): number {
     return Math.ceil(this.collectionSize / this.currentPageSize)
@@ -317,7 +323,7 @@ export class DocumentListViewService {
     return this.documents.map(d => d.id).indexOf(documentID)
   }
 
-  constructor(private documentService: DocumentService, private settings: SettingsService, private router: Router) {
+  constructor(private documentService: DocumentService, private settings: SettingsService, private router: Router, private route: ActivatedRoute) {
      let documentListViewConfigJson = sessionStorage.getItem(DOCUMENT_LIST_SERVICE.CURRENT_VIEW_CONFIG)
     if (documentListViewConfigJson) {
       try {

src-ui/src/app/services/rest/search.service.ts

@@ -2,8 +2,6 @@ import { HttpClient, HttpParams } from '@angular/common/http';
 import { Injectable } from '@angular/core';
 import { Observable } from 'rxjs';
 import { map } from 'rxjs/operators';
-import { PaperlessDocument } from 'src/app/data/paperless-document';
-import { SearchResult } from 'src/app/data/search-result';
 import { environment } from 'src/environments/environment';
 import { DocumentService } from './document.service';
 
@@ -13,30 +11,7 @@ import { DocumentService } from './document.service';
 })
 export class SearchService {
   
-  constructor(private http: HttpClient, private documentService: DocumentService) { }
+  constructor(private http: HttpClient) { }
-
-  search(query: string, page?: number, more_like?: number): Observable<SearchResult> {
-    let httpParams = new HttpParams()
-    if (query) {
-      httpParams = httpParams.set('query', query)
-    }
-    if (page) {
-      httpParams = httpParams.set('page', page.toString())
-    }
-    if (more_like) {
-      httpParams = httpParams.set('more_like', more_like.toString())
-    }
-    return this.http.get<SearchResult>(`${environment.apiBaseUrl}search/`, {params: httpParams}).pipe(
-      map(result => {
-        result.results.forEach(hit => {
-          if (hit.document) {
-            this.documentService.addObservablesToDocument(hit.document)
-          }
-        })
-        return result
-      })
-    )
-  }
 
   autocomplete(term: string): Observable<string[]> {
     return this.http.get<string[]>(`${environment.apiBaseUrl}search/autocomplete/`, {params: new HttpParams().set('term', term)})

src/documents/index.py

@@ -5,12 +5,12 @@ from contextlib import contextmanager
 import math
 from django.conf import settings
 from whoosh import highlight, classify, query
-from whoosh.fields import Schema, TEXT, NUMERIC, KEYWORD, DATETIME
+from whoosh.fields import Schema, TEXT, NUMERIC, KEYWORD, DATETIME, BOOLEAN
-from whoosh.highlight import Formatter, get_text
+from whoosh.highlight import Formatter, get_text, HtmlFormatter
 from whoosh.index import create_in, exists_in, open_dir
 from whoosh.qparser import MultifieldParser
 from whoosh.qparser.dateparse import DateParserPlugin
-from whoosh.searching import ResultsPage
+from whoosh.searching import ResultsPage, Searcher
 from whoosh.writing import AsyncWriter
 
 from documents.models import Document
@@ -18,63 +18,53 @@ from documents.models import Document
 logger = logging.getLogger("paperless.index")
 
 
-class JsonFormatter(Formatter):
-    def __init__(self):
-        self.seen = {}
-
-    def format_token(self, text, token, replace=False):
-        ttext = self._text(get_text(text, token, replace))
-        return {'text': ttext, 'highlight': 'true'}
-
-    def format_fragment(self, fragment, replace=False):
-        output = []
-        index = fragment.startchar
-        text = fragment.text
-        amend_token = None
-        for t in fragment.matches:
-            if t.startchar is None:
-                continue
-            if t.startchar < index:
-                continue
-            if t.startchar > index:
-                text_inbetween = text[index:t.startchar]
-                if amend_token and t.startchar - index < 10:
-                    amend_token['text'] += text_inbetween
-                else:
-                    output.append({'text': text_inbetween,
-                                   'highlight': False})
-                    amend_token = None
-            token = self.format_token(text, t, replace)
-            if amend_token:
-                amend_token['text'] += token['text']
-            else:
-                output.append(token)
-                amend_token = token
-            index = t.endchar
-        if index < fragment.endchar:
-            output.append({'text': text[index:fragment.endchar],
-                           'highlight': False})
-        return output
-
-    def format(self, fragments, replace=False):
-        output = []
-        for fragment in fragments:
-            output.append(self.format_fragment(fragment, replace=replace))
-        return output
-
-
 def get_schema():
     return Schema(
-        id=NUMERIC(stored=True, unique=True, numtype=int),
+        id=NUMERIC(
-        title=TEXT(stored=True),
+            stored=True,
+            unique=True
+        ),
+        title=TEXT(
+            sortable=True
+        ),
         content=TEXT(),
-        correspondent=TEXT(stored=True),
+        archive_serial_number=NUMERIC(
-        correspondent_id=NUMERIC(stored=True, numtype=int),
+            sortable=True
-        tag=KEYWORD(stored=True, commas=True, scorable=True, lowercase=True),
+        ),
-        type=TEXT(stored=True),
+
-        created=DATETIME(stored=True, sortable=True),
+        correspondent=TEXT(
-        modified=DATETIME(stored=True, sortable=True),
+            sortable=True
-        added=DATETIME(stored=True, sortable=True),
+        ),
+        correspondent_id=NUMERIC(),
+        has_correspondent=BOOLEAN(),
+
+        tag=KEYWORD(
+            commas=True,
+            scorable=True,
+            lowercase=True
+        ),
+        tag_id=KEYWORD(
+            commas=True,
+            scorable=True
+        ),
+        has_tag=BOOLEAN(),
+
+        type=TEXT(
+            sortable=True
+        ),
+        type_id=NUMERIC(),
+        has_type=BOOLEAN(),
+
+        created=DATETIME(
+            sortable=True
+        ),
+        modified=DATETIME(
+            sortable=True
+        ),
+        added=DATETIME(
+            sortable=True
+        ),
+
     )
 
 
@@ -106,18 +96,38 @@ def open_index_writer(ix=None, optimize=False):
         writer.commit(optimize=optimize)
 
 
+@contextmanager
+def open_index_searcher(ix=None):
+    if ix:
+        searcher = ix.searcher()
+    else:
+        searcher = open_index().searcher()
+
+    try:
+        yield searcher
+    finally:
+        searcher.close()
+
+
 def update_document(writer, doc):
     tags = ",".join([t.name for t in doc.tags.all()])
+    tags_ids = ",".join([str(t.id) for t in doc.tags.all()])
     writer.update_document(
         id=doc.pk,
         title=doc.title,
         content=doc.content,
         correspondent=doc.correspondent.name if doc.correspondent else None,
         correspondent_id=doc.correspondent.id if doc.correspondent else None,
+        has_correspondent=doc.correspondent is not None,
         tag=tags if tags else None,
+        tag_id=tags_ids if tags_ids else None,
+        has_tag=len(tags) > 0,
         type=doc.document_type.name if doc.document_type else None,
+        type_id=doc.document_type.id if doc.document_type else None,
+        has_type=doc.document_type is not None,
         created=doc.created,
         added=doc.added,
+        archive_serial_number=doc.archive_serial_number,
         modified=doc.modified,
     )
 
@@ -140,63 +150,111 @@ def remove_document_from_index(document):
         remove_document(writer, document)
 
 
-@contextmanager
-def query_page(ix, page, querystring, more_like_doc_id, more_like_doc_content):
-    searcher = ix.searcher()
-    try:
-        if querystring:
-            qp = MultifieldParser(
-                ["content", "title", "correspondent", "tag", "type"],
-                ix.schema)
-            qp.add_plugin(DateParserPlugin())
-            str_q = qp.parse(querystring)
-            corrected = searcher.correct_query(str_q, querystring)
-        else:
-            str_q = None
-            corrected = None
-
-        if more_like_doc_id:
-            docnum = searcher.document_number(id=more_like_doc_id)
-            kts = searcher.key_terms_from_text(
-                'content', more_like_doc_content, numterms=20,
-                model=classify.Bo1Model, normalize=False)
-            more_like_q = query.Or(
-                [query.Term('content', word, boost=weight)
-                 for word, weight in kts])
-            result_page = searcher.search_page(
-                more_like_q, page, filter=str_q, mask={docnum})
-        elif str_q:
-            result_page = searcher.search_page(str_q, page)
-        else:
-            raise ValueError(
-                "Either querystring or more_like_doc_id is required."
-            )
-
-        result_page.results.fragmenter = highlight.ContextFragmenter(
-            surround=50)
-        result_page.results.formatter = JsonFormatter()
-
-        if corrected and corrected.query != str_q:
-            corrected_query = corrected.string
-        else:
-            corrected_query = None
-
-        yield result_page, corrected_query
-    finally:
-        searcher.close()
-
-
 class DelayedQuery:
 
     @property
     def _query(self):
-        if 'query' in self.query_params:
+        raise NotImplementedError()
+
+    @property
+    def _query_filter(self):
+        criterias = []
+        for k, v in self.query_params.items():
+            if k == 'correspondent__id':
+                criterias.append(query.Term('correspondent_id', v))
+            elif k == 'tags__id__all':
+                for tag_id in v.split(","):
+                    criterias.append(query.Term('tag_id', tag_id))
+            elif k == 'document_type__id':
+                criterias.append(query.Term('type_id', v))
+            elif k == 'correspondent__isnull':
+                criterias.append(query.Term("has_correspondent", v == "false"))
+            elif k == 'is_tagged':
+                criterias.append(query.Term("has_tag", v == "true"))
+            elif k == 'document_type__isnull':
+                criterias.append(query.Term("has_type", v == "false"))
+            elif k == 'created__date__lt':
+                pass
+            elif k == 'created__date__gt':
+                pass
+            elif k == 'added__date__gt':
+                pass
+            elif k == 'added__date__lt':
+                pass
+        if len(criterias) > 0:
+            return query.And(criterias)
+        else:
+            return None
+
+    @property
+    def _query_sortedby(self):
+        if not 'ordering' in self.query_params:
+            return None, False
+
+        o: str = self.query_params['ordering']
+        if o.startswith('-'):
+            return o[1:], True
+        else:
+            return o, False
+
+    def __init__(self, searcher: Searcher, query_params, page_size):
+        self.searcher = searcher
+        self.query_params = query_params
+        self.page_size = page_size
+        self.saved_results = dict()
+
+    def __len__(self):
+        page = self[0:1]
+        return len(page)
+
+    def __getitem__(self, item):
+        if item.start in self.saved_results:
+            return self.saved_results[item.start]
+
+        q, mask = self._query
+        sortedby, reverse = self._query_sortedby
+
+        print("OY", self.page_size)
+        page: ResultsPage = self.searcher.search_page(
+            q,
+            mask=mask,
+            filter=self._query_filter,
+            pagenum=math.floor(item.start / self.page_size) + 1,
+            pagelen=self.page_size,
+            sortedby=sortedby,
+            reverse=reverse
+        )
+        page.results.fragmenter = highlight.ContextFragmenter(
+            surround=50)
+        page.results.formatter = HtmlFormatter(tagname="span", between=" ... ")
+
+        self.saved_results[item.start] = page
+
+        return page
+
+
+class DelayedFullTextQuery(DelayedQuery):
+
+    @property
+    def _query(self):
+        q_str = self.query_params['query']
         qp = MultifieldParser(
             ["content", "title", "correspondent", "tag", "type"],
-                self.ix.schema)
+            self.searcher.ixreader.schema)
         qp.add_plugin(DateParserPlugin())
-            q = qp.parse(self.query_params['query'])
+        q = qp.parse(q_str)
-        elif 'more_like_id' in self.query_params:
+
+        corrected = self.searcher.correct_query(q, q_str)
+        if corrected.query != q:
+            corrected_query = corrected.string
+
+        return q, None
+
+
+class DelayedMoreLikeThisQuery(DelayedQuery):
+
+    @property
+    def _query(self):
         more_like_doc_id = int(self.query_params['more_like_id'])
         content = Document.objects.get(id=more_like_doc_id).content
 
@@ -207,42 +265,9 @@ class DelayedQuery:
         q = query.Or(
             [query.Term('content', word, boost=weight)
              for word, weight in kts])
-        else:
+        mask = {docnum}
-            raise ValueError(
-                "Either query or more_like_id is required."
-            )
-        return q
 
-    @property
+        return q, mask
-    def _query_filter(self):
-        criterias = []
-        for k, v in self.query_params.items():
-            if k == 'correspondent__id':
-                criterias.append(query.Term('correspondent_id', v))
-        if len(criterias) > 0:
-            return query.And(criterias)
-        else:
-            return None
-
-    def __init__(self, ix, searcher, query_params, page_size):
-        self.ix = ix
-        self.searcher = searcher
-        self.query_params = query_params
-        self.page_size = page_size
-
-    def __len__(self):
-        results = self.searcher.search(self._query, limit=1, filter=self._query_filter)
-        return len(results)
-        #return 1000
-
-    def __getitem__(self, item):
-        page: ResultsPage = self.searcher.search_page(
-            self._query,
-            filter=self._query_filter,
-            pagenum=math.floor(item.start / self.page_size) + 1,
-            pagelen=self.page_size
-        )
-        return page
 
 
 def autocomplete(ix, term, limit=10):

src/documents/models.py

@@ -359,7 +359,10 @@ class SavedView(models.Model):
 
     sort_field = models.CharField(
         _("sort field"),
-        max_length=128)
+        max_length=128,
+        null=True,
+        blank=True
+    )
     sort_reverse = models.BooleanField(
         _("sort reverse"),
         default=False)
@@ -387,6 +390,8 @@ class SavedViewFilterRule(models.Model):
         (17, _("does not have tag")),
         (18, _("does not have ASN")),
         (19, _("title or content contains")),
+        (20, _("fulltext query")),
+        (21, _("more like this"))
     ]
 
     saved_view = models.ForeignKey(

src/documents/tests/test_index.py

@@ -1,20 +1,10 @@
 from django.test import TestCase
 
 from documents import index
-from documents.index import JsonFormatter
 from documents.models import Document
 from documents.tests.utils import DirectoriesMixin
 
 
-class JsonFormatterTest(TestCase):
-
-    def setUp(self) -> None:
-        self.formatter = JsonFormatter()
-
-    def test_empty_fragments(self):
-        self.assertListEqual(self.formatter.format([]), [])
-
-
 class TestAutoComplete(DirectoriesMixin, TestCase):
 
     def test_auto_complete(self):

src/documents/views.py

@@ -36,7 +36,6 @@ from rest_framework.viewsets import (
 
 from paperless.db import GnuPG
 from paperless.views import StandardPagination
-from . import index
 from .bulk_download import OriginalAndArchiveStrategy, OriginalsOnlyStrategy, \
     ArchiveOnlyStrategy
 from .classifier import load_classifier
@@ -332,15 +331,23 @@ class SearchResultSerializer(DocumentSerializer):
 
     def to_representation(self, instance):
         doc = Document.objects.get(id=instance['id'])
-        # repressentation = super(SearchResultSerializer, self).to_representation(doc)
+        representation = super(SearchResultSerializer, self).to_representation(doc)
-        # repressentation['__search_hit__'] = {
+        representation['__search_hit__'] = {
-        #     "score": instance.score
+            "score": instance.score,
-        # }
+            "highlights": instance.highlights("content",
-        return super(SearchResultSerializer, self).to_representation(doc)
+                                   text=doc.content) if doc else None,  # NOQA: E501
+            "rank": instance.rank
+        }
+
+        return representation
 
 
 class UnifiedSearchViewSet(DocumentViewSet):
 
+    def __init__(self, *args, **kwargs):
+        super(UnifiedSearchViewSet, self).__init__(*args, **kwargs)
+        self.searcher = None
+
     def get_serializer_class(self):
         if self._is_search_request():
             return SearchResultSerializer
@@ -348,25 +355,39 @@ class UnifiedSearchViewSet(DocumentViewSet):
             return DocumentSerializer
 
     def _is_search_request(self):
-        return "query" in self.request.query_params
+        return "query" in self.request.query_params or "more_like_id" in self.request.query_params
 
     def filter_queryset(self, queryset):
-
         if self._is_search_request():
-            ix = index.open_index()
+            from documents import index
-            return index.DelayedQuery(ix, self.searcher, self.request.query_params, self.paginator.page_size)
+
+            if "query" in self.request.query_params:
+                query_class = index.DelayedFullTextQuery
+            elif "more_like_id" in self.request.query_params:
+                query_class = index.DelayedMoreLikeThisQuery
+            else:
+                raise ValueError()
+
+            return query_class(
+                self.searcher,
+                self.request.query_params,
+                self.paginator.get_page_size(self.request))
         else:
             return super(UnifiedSearchViewSet, self).filter_queryset(queryset)
 
     def list(self, request, *args, **kwargs):
         if self._is_search_request():
-            ix = index.open_index()
+            from documents import index
-            with ix.searcher() as s:
+            try:
+                with index.open_index_searcher() as s:
                     self.searcher = s
                     return super(UnifiedSearchViewSet, self).list(request)
+            except Exception as e:
+                return HttpResponseBadRequest(str(e))
         else:
             return super(UnifiedSearchViewSet, self).list(request)
 
+
 class LogViewSet(ViewSet):
 
     permission_classes = (IsAuthenticated,)
@@ -518,74 +539,6 @@ class SelectionDataView(GenericAPIView):
         return r
 
 
-class SearchView(APIView):
-
-    permission_classes = (IsAuthenticated,)
-
-    def add_infos_to_hit(self, r):
-        try:
-            doc = Document.objects.get(id=r['id'])
-        except Document.DoesNotExist:
-            logger.warning(
-                f"Search index returned a non-existing document: "
-                f"id: {r['id']}, title: {r['title']}. "
-                f"Search index needs reindex."
-            )
-            doc = None
-
-        return {'id': r['id'],
-                'highlights': r.highlights("content", text=doc.content) if doc else None,  # NOQA: E501
-                'score': r.score,
-                'rank': r.rank,
-                'document': DocumentSerializer(doc).data if doc else None,
-                'title': r['title']
-                }
-
-    def get(self, request, format=None):
-        from documents import index
-
-        if 'query' in request.query_params:
-            query = request.query_params['query']
-        else:
-            query = None
-
-        if 'more_like' in request.query_params:
-            more_like_id = request.query_params['more_like']
-            more_like_content = Document.objects.get(id=more_like_id).content
-        else:
-            more_like_id = None
-            more_like_content = None
-
-        if not query and not more_like_id:
-            return Response({
-                'count': 0,
-                'page': 0,
-                'page_count': 0,
-                'corrected_query': None,
-                'results': []})
-
-        try:
-            page = int(request.query_params.get('page', 1))
-        except (ValueError, TypeError):
-            page = 1
-
-        if page < 1:
-            page = 1
-
-        ix = index.open_index()
-
-        try:
-            with index.query_page(ix, page, query, more_like_id, more_like_content) as (result_page, corrected_query):  # NOQA: E501
-                return Response(
-                    {'count': len(result_page),
-                     'page': result_page.pagenum,
-                     'page_count': result_page.pagecount,
-                     'corrected_query': corrected_query,
-                     'results': list(map(self.add_infos_to_hit, result_page))})
-        except Exception as e:
-            return HttpResponseBadRequest(str(e))
-
-
 class SearchAutoCompleteView(APIView):
 
     permission_classes = (IsAuthenticated,)

src/paperless/urls.py

@@ -16,7 +16,6 @@ from documents.views import (
     LogViewSet,
     TagViewSet,
     DocumentTypeViewSet,
-    SearchView,
     IndexView,
     SearchAutoCompleteView,
     StatisticsView,
@@ -47,10 +46,6 @@ urlpatterns = [
                 SearchAutoCompleteView.as_view(),
                 name="autocomplete"),
 
-        re_path(r"^search/",
-                SearchView.as_view(),
-                name="search"),
-
         re_path(r"^statistics/",
                 StatisticsView.as_view(),
                 name="statistics"),