001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021
022import java.io.IOException;
023import java.nio.ByteBuffer;
024import java.util.ArrayList;
025import java.util.List;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.Cell;
028import org.apache.hadoop.hbase.HBaseTestingUtil;
029import org.apache.hadoop.hbase.HConstants;
030import org.apache.hadoop.hbase.TableName;
031import org.apache.hadoop.hbase.client.Durability;
032import org.apache.hadoop.hbase.client.Put;
033import org.apache.hadoop.hbase.client.Scan;
034import org.apache.hadoop.hbase.client.Table;
035import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
036import org.apache.hadoop.hbase.regionserver.HRegion;
037import org.apache.hadoop.hbase.regionserver.RegionScanner;
038import org.apache.hadoop.hbase.testclassification.FilterTests;
039import org.apache.hadoop.hbase.testclassification.LargeTests;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
042import org.apache.hadoop.hbase.util.Pair;
043import org.junit.jupiter.api.AfterAll;
044import org.junit.jupiter.api.BeforeAll;
045import org.junit.jupiter.api.Tag;
046import org.junit.jupiter.api.Test;
047import org.slf4j.Logger;
048import org.slf4j.LoggerFactory;
049
050@Tag(FilterTests.TAG)
051@Tag(LargeTests.TAG)
052public class TestFuzzyRowFilterEndToEndLarge {
053
054  private static final Logger LOG = LoggerFactory.getLogger(TestFuzzyRowFilterEndToEndLarge.class);
055
056  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
057
058  private final static byte fuzzyValue = (byte) 63;
059
060  private static int firstPartCardinality = 30;
061  private static int secondPartCardinality = 30;
062  private static int thirdPartCardinality = 30;
063  private static int colQualifiersTotal = 5;
064  private static int totalFuzzyKeys = thirdPartCardinality / 2;
065
066  private static String table = "TestFuzzyRowFilterEndToEndLarge";
067
068  @BeforeAll
069  public static void setUpBeforeClass() throws Exception {
070    Configuration conf = TEST_UTIL.getConfiguration();
071    conf.setInt("hbase.client.scanner.caching", 1000);
072    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
073      ConstantSizeRegionSplitPolicy.class.getName());
074    // set no splits
075    conf.setLong(HConstants.HREGION_MAX_FILESIZE, (1024L) * 1024 * 1024 * 10);
076
077    TEST_UTIL.startMiniCluster();
078  }
079
080  @AfterAll
081  public static void tearDownAfterClass() throws Exception {
082    TEST_UTIL.shutdownMiniCluster();
083  }
084
085  @Test
086  public void testEndToEnd() throws Exception {
087    String cf = "f";
088
089    Table ht =
090      TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
091
092    // 10 byte row key - (2 bytes 4 bytes 4 bytes)
093    // 4 byte qualifier
094    // 4 byte value
095
096    for (int i0 = 0; i0 < firstPartCardinality; i0++) {
097      for (int i1 = 0; i1 < secondPartCardinality; i1++) {
098        for (int i2 = 0; i2 < thirdPartCardinality; i2++) {
099          byte[] rk = new byte[10];
100
101          ByteBuffer buf = ByteBuffer.wrap(rk);
102          buf.clear();
103          buf.putShort((short) i0);
104          buf.putInt(i1);
105          buf.putInt(i2);
106          for (int c = 0; c < colQualifiersTotal; c++) {
107            byte[] cq = new byte[4];
108            Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
109
110            Put p = new Put(rk);
111            p.setDurability(Durability.SKIP_WAL);
112            p.addColumn(Bytes.toBytes(cf), cq, Bytes.toBytes(c));
113            ht.put(p);
114          }
115        }
116      }
117    }
118
119    TEST_UTIL.flush();
120
121    // test passes
122    runTest1(ht);
123    runTest2(ht);
124
125  }
126
127  private void runTest1(Table hTable) throws IOException {
128    // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]
129    byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
130
131    List<Pair<byte[], byte[]>> list = new ArrayList<>();
132    for (int i = 0; i < totalFuzzyKeys; i++) {
133      byte[] fuzzyKey = new byte[10];
134      ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
135      buf.clear();
136      buf.putShort((short) 2);
137      for (int j = 0; j < 4; j++) {
138        buf.put(fuzzyValue);
139      }
140      buf.putInt(i);
141
142      Pair<byte[], byte[]> pair = new Pair<>(fuzzyKey, mask);
143      list.add(pair);
144    }
145
146    int expectedSize = secondPartCardinality * totalFuzzyKeys * colQualifiersTotal;
147    FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
148    // Filters are not stateless - we can't reuse them
149    FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);
150
151    // regular test
152    runScanner(hTable, expectedSize, fuzzyRowFilter0);
153    // optimized from block cache
154    runScanner(hTable, expectedSize, fuzzyRowFilter1);
155
156  }
157
158  private void runTest2(Table hTable) throws IOException {
159    // [0, 0, ?, ?, ?, ?, 0, 0, 0, 0] , [0, 1, ?, ?, ?, ?, 0, 0, 0, 1]...
160    byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
161
162    List<Pair<byte[], byte[]>> list = new ArrayList<>();
163
164    for (int i = 0; i < totalFuzzyKeys; i++) {
165      byte[] fuzzyKey = new byte[10];
166      ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
167      buf.clear();
168      buf.putShort((short) (i * 2));
169      for (int j = 0; j < 4; j++) {
170        buf.put(fuzzyValue);
171      }
172      buf.putInt(i * 2);
173
174      Pair<byte[], byte[]> pair = new Pair<>(fuzzyKey, mask);
175      list.add(pair);
176    }
177
178    int expectedSize = totalFuzzyKeys * secondPartCardinality * colQualifiersTotal;
179
180    FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
181    // Filters are not stateless - we can't reuse them
182    FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);
183
184    // regular test
185    runScanner(hTable, expectedSize, fuzzyRowFilter0);
186    // optimized from block cache
187    runScanner(hTable, expectedSize, fuzzyRowFilter1);
188
189  }
190
191  private void runScanner(Table hTable, int expectedSize, Filter filter) throws IOException {
192    String cf = "f";
193    Scan scan = new Scan();
194    scan.addFamily(Bytes.toBytes(cf));
195    scan.setFilter(filter);
196    List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(TableName.valueOf(table));
197    HRegion first = regions.get(0);
198    first.getScanner(scan);
199    RegionScanner scanner = first.getScanner(scan);
200    List<Cell> results = new ArrayList<>();
201    // Result result;
202    long timeBeforeScan = EnvironmentEdgeManager.currentTime();
203    int found = 0;
204    while (scanner.next(results)) {
205      found += results.size();
206      results.clear();
207    }
208    found += results.size();
209    long scanTime = EnvironmentEdgeManager.currentTime() - timeBeforeScan;
210    scanner.close();
211
212    LOG.info("\nscan time = " + scanTime + "ms");
213    LOG.info("found " + found + " results\n");
214
215    assertEquals(expectedSize, found);
216  }
217}