Remove overdue deprecated SortField optimization toggle methods

Remove setOptimizeSortWithIndexedData, getOptimizeSortWithIndexedData,
setOptimizeSortWithPoints, and getOptimizeSortWithPoints from SortField.
These methods were marked '@deprecated // Remove in Lucene 10' for
compatibility with 8.x indices. Since we are now on Lucene 11, sort
optimization is unconditionally enabled.

Also cleaned up the corresponding disableSkipping() calls in
SortedNumericSortField and SortedSetSortField, and removed test paths
in TestSortOptimization that exercised the disabled-optimization mode.

Author: Bharathi-Kanna

lucene/CHANGES.txt

@@ -235,6 +235,9 @@ Build
 Other
 ---------------------
 
+* GITHUB#XXXXX: Removed overdue deprecated methods setOptimizeSortWithIndexedData and
+  setOptimizeSortWithPoints (along with their getters) from SortField. Sort optimization
+  is now unconditionally enabled when possible. (Bharathi Kanna)
 * GITHUB#16074: Ban presized collection#toArray. (Zhou Hui)
 
 * GITHUB#16063: Remove redundant final modifiers. (Zhou Hui)

lucene/core/src/java/org/apache/lucene/search/SortField.java

@@ -133,9 +133,6 @@ public enum Type {
   // Used for 'sortMissingFirst/Last'
   protected final Object missingValue;
 
-  // Indicates if sort should be optimized with indexed data. Set to true by default.
-  @Deprecated private boolean optimizeSortWithIndexedData = true;
-
   /**
    * Creates a sort by terms in the given field with the type of term values explicitly given.
    *
@@ -530,9 +527,7 @@ public FieldComparator<?> getComparator(final int numHits, Pruning pruning) {
       default:
         throw new IllegalStateException("Illegal sort type: " + type);
     }
-    if (getOptimizeSortWithIndexedData() == false) {
-      fieldComparator.disableSkipping();
-    }
+
     return fieldComparator;
   }
 
@@ -604,68 +599,4 @@ public IndexSorter getIndexSorter() {
         return null;
     }
   }
-
-  /**
-   * Enables/disables numeric sort optimization to use the indexed data.
-   *
-   * <p>Enabled by default. By default, sorting on a numeric field activates point sort optimization
-   * that can efficiently skip over non-competitive hits. Sort optimization has a number of
-   * requirements, one of which is that SortField.Type matches the Point type with which the field
-   * was indexed (e.g. sort on IntPoint field should use SortField.Type.INT). Another requirement is
-   * that the same data is indexed with points and doc values for the field.
-   *
-   * <p>By default, sorting on a SORTED(_SET) field activates sort optimization that can efficiently
-   * skip over non-competitive hits. Sort optimization requires that the same data is indexed with
-   * term index and doc values for the field.
-   *
-   * @param optimizeSortWithIndexedData providing {@code false} disables the optimization, in cases
-   *     where these requirements can't be met.
-   * @deprecated should only be used for compatibility with 8.x indices that got created with
-   *     inconsistent data across fields, or the wrong sort configuration in the index sort
-   */
-  @Deprecated // Remove in Lucene 10
-  public void setOptimizeSortWithIndexedData(boolean optimizeSortWithIndexedData) {
-    this.optimizeSortWithIndexedData = optimizeSortWithIndexedData;
-  }
-
-  /**
-   * Returns whether sort optimization should be optimized with indexed data
-   *
-   * @return whether sort optimization should be optimized with indexed data
-   */
-  @Deprecated // Remove in Lucene 10
-  public boolean getOptimizeSortWithIndexedData() {
-    return optimizeSortWithIndexedData;
-  }
-
-  /**
-   * Enables/disables numeric sort optimization to use the Points index.
-   *
-   * <p>Enabled by default. By default, sorting on a numeric field activates point sort optimization
-   * that can efficiently skip over non-competitive hits. Sort optimization has a number of
-   * requirements, one of which is that SortField.Type matches the Point type with which the field
-   * was indexed (e.g. sort on IntPoint field should use SortField.Type.INT). Another requirement is
-   * that the same data is indexed with points and doc values for the field.
-   *
-   * @param optimizeSortWithPoints providing {@code false} disables the optimization, in cases where
-   *     these requirements can't be met.
-   * @deprecated should only be used for compatibility with 8.x indices that got created with
-   *     inconsistent data across fields, or the wrong sort configuration in the index sort. This is
-   *     a duplicate method for {@code SortField#setOptimizeSortWithIndexedData}.
-   */
-  @Deprecated // Remove in Lucene 10
-  public void setOptimizeSortWithPoints(boolean optimizeSortWithPoints) {
-    setOptimizeSortWithIndexedData(optimizeSortWithPoints);
-  }
-
-  /**
-   * Returns whether sort optimization should be optimized with points index
-   *
-   * @return whether sort optimization should be optimized with points index
-   * @deprecated This is a duplicate method for {@code SortField#getOptimizeSortWithIndexedData}.
-   */
-  @Deprecated // Remove in Lucene 10
-  public boolean getOptimizeSortWithPoints() {
-    return getOptimizeSortWithIndexedData();
-  }
 }

lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java

@@ -359,9 +359,7 @@ protected NumericDocValues getNumericDocValues(
       default:
         throw new AssertionError();
     }
-    if (getOptimizeSortWithIndexedData() == false) {
-      fieldComparator.disableSkipping();
-    }
+
     return fieldComparator;
   }
 

lucene/core/src/java/org/apache/lucene/search/SortedSetSortField.java

@@ -171,9 +171,8 @@ public String toString() {
 
   @Override
   public FieldComparator<?> getComparator(int numHits, Pruning pruning) {
-    Pruning finalPruning = getOptimizeSortWithIndexedData() ? pruning : Pruning.NONE;
     return new TermOrdValComparator(
-        numHits, getField(), missingValue == STRING_LAST, reverse, finalPruning) {
+        numHits, getField(), missingValue == STRING_LAST, reverse, pruning) {
       @Override
       protected SortedDocValues getSortedDocValues(LeafReaderContext context, String field)
           throws IOException {

lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java

@@ -331,33 +331,11 @@ private void testNumericDocValuesOptimizationWithMissingValues(
     final DirectoryReader reader = DirectoryReader.open(writer);
     writer.close();
     final int numHits = 3;
-    TopDocs topDocs1;
-    TopDocs topDocs2;
-
     { // Test that optimization is run with NumericDocValues when missing value is NOT competitive
       final SortField sortField = new SortField("my_field", SortField.Type.LONG, true, 0L);
       final Sort sort = new Sort(sortField);
-      topDocs1 = assertSearchHits(reader, sort, numHits, null);
-      assertNonCompetitiveHitsAreSkipped(topDocs1.totalHits.value(), numDocs);
-    }
-    { // Test that sort on sorted numeric field without sort optimization and with sort optimization
-      // produce the same results
-      final SortField sortField = new SortField("my_field", SortField.Type.LONG, true, 0L);
-      final Sort sort = new Sort(sortField);
-      sortField.setOptimizeSortWithPoints(false);
-      topDocs2 = assertSearchHits(reader, sort, numHits, null);
-      // assert that the resulting hits are the same
-      assertEquals(topDocs1.scoreDocs.length, topDocs2.scoreDocs.length);
-      assertEquals(topDocs1.scoreDocs.length, numHits);
-      ScoreDoc[] scoreDocs1 = topDocs1.scoreDocs;
-      ScoreDoc[] scoreDocs2 = topDocs2.scoreDocs;
-      for (int i = 0; i < numHits; i++) {
-        FieldDoc fieldDoc = (FieldDoc) scoreDocs1[i];
-        FieldDoc fieldDoc2 = (FieldDoc) scoreDocs2[i];
-        assertEquals(fieldDoc.fields[0], fieldDoc2.fields[0]);
-        assertEquals(fieldDoc.doc, fieldDoc2.doc);
-      }
-      assertTrue(topDocs1.totalHits.value() < topDocs2.totalHits.value());
+      TopDocs topDocs = assertSearchHits(reader, sort, numHits, null);
+      assertNonCompetitiveHitsAreSkipped(topDocs.totalHits.value(), numDocs);
     }
 
     { // Test that we can't do optimization via NumericDocValues when there are multiple comparators
@@ -795,25 +773,15 @@ public void testPointValidation() throws IOException {
     assertThrows(
         IllegalArgumentException.class,
         () -> searcher.search(MatchAllDocsQuery.INSTANCE, 1, new Sort(longSortOnIntField)));
-    // assert that when sort optimization is disabled we can use LONG sort on int field
-    longSortOnIntField.setOptimizeSortWithIndexedData(false);
-    searcher.search(MatchAllDocsQuery.INSTANCE, 1, new Sort(longSortOnIntField));
-
     SortField intSortOnLongField = new SortField("longField", SortField.Type.INT);
     assertThrows(
         IllegalArgumentException.class,
         () -> searcher.search(MatchAllDocsQuery.INSTANCE, 1, new Sort(intSortOnLongField)));
-    // assert that when sort optimization is disabled we can use INT sort on long field
-    intSortOnLongField.setOptimizeSortWithIndexedData(false);
-    searcher.search(MatchAllDocsQuery.INSTANCE, 1, new Sort(intSortOnLongField));
 
     SortField intSortOnIntRangeField = new SortField("intRange", SortField.Type.INT);
     assertThrows(
         IllegalArgumentException.class,
         () -> searcher.search(MatchAllDocsQuery.INSTANCE, 1, new Sort(intSortOnIntRangeField)));
-    // assert that when sort optimization is disabled we can use INT sort on intRange field
-    intSortOnIntRangeField.setOptimizeSortWithIndexedData(false);
-    searcher.search(MatchAllDocsQuery.INSTANCE, 1, new Sort(intSortOnIntRangeField));
 
     reader.close();
     dir.close();
@@ -948,14 +916,8 @@ public void testSortOptimizationOnSortedNumericField() throws IOException {
         RandomPicks.randomFrom(random(), SortedNumericSelector.Type.values());
     boolean reverse = random().nextBoolean();
     final SortField sortField = LongField.newSortField("my_field", reverse, type);
-    sortField.setOptimizeSortWithIndexedData(false);
-    final Sort sort = new Sort(sortField); // sort without sort optimization
-    final SortField sortField2 = LongField.newSortField("my_field", reverse, type);
-    final Sort sort2 = new Sort(sortField2); // sort with sort optimization
-
-    long expectedCollectedHits = 0;
-    long collectedHits = 0;
-    long collectedHits2 = 0;
+    final Sort sort = new Sort(sortField); // sort with sort optimization
+
     int visitedHits = 0;
     FieldDoc after = null;
     while (visitedHits < numDocs) {
@@ -965,30 +927,14 @@ public void testSortOptimizationOnSortedNumericField() throws IOException {
       TopDocs topDocs = assertSearchHits(reader, sort, batch, after);
       ScoreDoc[] scoreDocs = topDocs.scoreDocs;
 
-      TopDocs topDocs2 = assertSearchHits(reader, sort2, batch, after);
-      ScoreDoc[] scoreDocs2 = topDocs2.scoreDocs;
-
-      // assert that the resulting hits are the same
       assertEquals(expectedHits, topDocs.scoreDocs.length);
-      assertEquals(topDocs.scoreDocs.length, topDocs2.scoreDocs.length);
       for (int i = 0; i < scoreDocs.length; i++) {
-        FieldDoc fieldDoc = (FieldDoc) scoreDocs[i];
-        FieldDoc fieldDoc2 = (FieldDoc) scoreDocs2[i];
-        assertEquals(fieldDoc.fields[0], fieldDoc2.fields[0]);
-        assertEquals(fieldDoc.doc, fieldDoc2.doc);
         visitedHits++;
       }
 
-      expectedCollectedHits += numDocs;
-      collectedHits += topDocs.totalHits.value();
-      collectedHits2 += topDocs2.totalHits.value();
       after = (FieldDoc) scoreDocs[expectedHits - 1];
     }
     assertEquals(visitedHits, numDocs);
-    assertEquals(expectedCollectedHits, collectedHits);
-    // assert that the second sort with optimization collected less or equal hits
-    assertTrue(collectedHits >= collectedHits2);
-    // System.out.println(expectedCollectedHits + "\t" + collectedHits + "\t" + collectedHits2);
 
     reader.close();
     dir.close();
@@ -1099,7 +1045,7 @@ private void testStringSortOptimization(
     final DirectoryReader reader = DirectoryReader.open(writer);
     writer.close();
     doTestStringSortOptimization(reader);
-    doTestStringSortOptimizationDisabled(reader);
+
     reader.close();
     dir.close();
   }
@@ -1203,12 +1149,6 @@ private void testStringSortOptimizationFieldMissingInSegment(
               "my_field", true, SortedSetSelector.Type.MIN, SortField.STRING_LAST);
       Sort sort = new Sort(sortField);
       TopDocs topDocs = assertSearchHits(reader, sort, numHits, null);
-
-      sortField.setOptimizeSortWithIndexedData(false);
-      sort = new Sort(sortField);
-      TopDocs unpruned = assertSearchHits(reader, sort, numHits, null);
-
-      CheckHits.checkEqual(MatchAllDocsQuery.INSTANCE, topDocs.scoreDocs, unpruned.scoreDocs);
     }
 
     { // descending sort with missing-first: once the queue fills from the first segment,
@@ -1227,12 +1167,6 @@ private void testStringSortOptimizationFieldMissingInSegment(
               "my_field", false, SortedSetSelector.Type.MIN, SortField.STRING_FIRST);
       Sort sort = new Sort(sortField);
       TopDocs topDocs = assertSearchHits(reader, sort, numHits, null);
-
-      sortField.setOptimizeSortWithIndexedData(false);
-      sort = new Sort(sortField);
-      TopDocs unpruned = assertSearchHits(reader, sort, numHits, null);
-
-      CheckHits.checkEqual(MatchAllDocsQuery.INSTANCE, topDocs.scoreDocs, unpruned.scoreDocs);
     }
 
     reader.close();
@@ -1342,20 +1276,6 @@ private void doTestStringSortOptimization(DirectoryReader reader) throws IOExcep
     }
   }
 
-  public void doTestStringSortOptimizationDisabled(DirectoryReader reader) throws IOException {
-    SortField sortField =
-        KeywordField.newSortField(
-            "my_field", false, SortedSetSelector.Type.MIN, SortField.STRING_LAST);
-    sortField.setOptimizeSortWithIndexedData(false);
-
-    Sort sort = new Sort(sortField);
-    final int numDocs = reader.numDocs();
-    final int numHits = 5;
-
-    TopDocs topDocs = assertSearchHits(reader, sort, numHits, null);
-    assertEquals(numDocs, topDocs.totalHits.value());
-  }
-
   private TopDocs assertSort(DirectoryReader reader, Sort sort, int n, FieldDoc after)
       throws IOException {
     TopDocs topDocs = assertSearchHits(reader, sort, n, after);