001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.jupiter.api.Assertions.assertTrue;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.Arrays;
025import java.util.List;
026import org.apache.commons.codec.binary.Hex;
027import org.apache.hadoop.hbase.Cell;
028import org.apache.hadoop.hbase.CellUtil;
029import org.apache.hadoop.hbase.CompareOperator;
030import org.apache.hadoop.hbase.HBaseTestingUtil;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.client.Put;
033import org.apache.hadoop.hbase.client.Result;
034import org.apache.hadoop.hbase.client.ResultScanner;
035import org.apache.hadoop.hbase.client.Scan;
036import org.apache.hadoop.hbase.client.Table;
037import org.apache.hadoop.hbase.testclassification.MediumTests;
038import org.apache.hadoop.hbase.util.Bytes;
039import org.junit.jupiter.api.AfterAll;
040import org.junit.jupiter.api.BeforeAll;
041import org.junit.jupiter.api.Tag;
042import org.junit.jupiter.api.Test;
043import org.junit.jupiter.api.TestInfo;
044import org.slf4j.Logger;
045import org.slf4j.LoggerFactory;
046
047@Tag(MediumTests.TAG)
048public class TestFiltersWithBinaryComponentComparator {
049
050  /**
051   * See https://issues.apache.org/jira/browse/HBASE-22969 - for need of BinaryComponentComparator
052   * The descrption on jira should also help you in understanding tests implemented in this class
053   */
054
055  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
056  private static final Logger LOG =
057    LoggerFactory.getLogger(TestFiltersWithBinaryComponentComparator.class);
058  private byte[] family = Bytes.toBytes("family");
059  private byte[] qf = Bytes.toBytes("qf");
060  private TableName tableName;
061  private int aOffset = 0;
062  private int bOffset = 4;
063  private int cOffset = 8;
064  private int dOffset = 12;
065
066  @BeforeAll
067  public static void setUpBeforeClass() throws Exception {
068    TEST_UTIL.startMiniCluster();
069  }
070
071  @AfterAll
072  public static void tearDownAfterClass() throws Exception {
073    TEST_UTIL.shutdownMiniCluster();
074  }
075
076  @Test
077  public void testRowFilterWithBinaryComponentComparator(TestInfo testInfo) throws IOException {
078    // SELECT * from table where a=1 and b > 10 and b < 20 and c > 90 and c < 100 and d=1
079    tableName = TableName.valueOf(testInfo.getTestMethod().get().getName());
080    Table ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE);
081    generateRows(ht, family, qf);
082    FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
083    setRowFilters(filterList);
084    Scan scan = createScan(filterList);
085    List<Cell> result = getResults(ht, scan);
086    for (Cell cell : result) {
087      byte[] key = CellUtil.cloneRow(cell);
088      int a = Bytes.readAsInt(key, aOffset, 4);
089      int b = Bytes.readAsInt(key, bOffset, 4);
090      int c = Bytes.readAsInt(key, cOffset, 4);
091      int d = Bytes.readAsInt(key, dOffset, 4);
092      assertTrue(a == 1 && b > 10 && b < 20 && c > 90 && c < 100 && d == 1);
093    }
094    ht.close();
095  }
096
097  @Test
098  public void testValueFilterWithBinaryComponentComparator(TestInfo testInfo) throws IOException {
099    // SELECT * from table where value has 'y' at position 1
100    tableName = TableName.valueOf(testInfo.getTestMethod().get().getName());
101    Table ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE);
102    generateRows(ht, family, qf);
103    FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
104    setValueFilters(filterList);
105    Scan scan = new Scan();
106    scan.setFilter(filterList);
107    List<Cell> result = getResults(ht, scan);
108    for (Cell cell : result) {
109      byte[] value = CellUtil.cloneValue(cell);
110      assertTrue(Bytes.toString(value).charAt(1) == 'y');
111    }
112    ht.close();
113  }
114
115  @Test
116  public void testRowAndValueFilterWithBinaryComponentComparator(TestInfo testInfo)
117    throws IOException {
118    // SELECT * from table where a=1 and b > 10 and b < 20 and c > 90 and c < 100 and d=1
119    // and value has 'y' at position 1"
120    tableName = TableName.valueOf(testInfo.getTestMethod().get().getName());
121    Table ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE);
122    generateRows(ht, family, qf);
123    FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
124    setRowFilters(filterList);
125    setValueFilters(filterList);
126    Scan scan = new Scan();
127    scan.setFilter(filterList);
128    List<Cell> result = getResults(ht, scan);
129    for (Cell cell : result) {
130      byte[] key = CellUtil.cloneRow(cell);
131      int a = Bytes.readAsInt(key, aOffset, 4);
132      int b = Bytes.readAsInt(key, bOffset, 4);
133      int c = Bytes.readAsInt(key, cOffset, 4);
134      int d = Bytes.readAsInt(key, dOffset, 4);
135      assertTrue(a == 1 && b > 10 && b < 20 && c > 90 && c < 100 && d == 1);
136      byte[] value = CellUtil.cloneValue(cell);
137      assertTrue(Bytes.toString(value).charAt(1) == 'y');
138    }
139    ht.close();
140  }
141
142  /**
143   * Since we are trying to emulate SQL: SELECT * from table where a = 1 and b > 10 and b < 20 and c
144   * > 90 and c < 100 and d = 1 We are generating rows with: a = 1, b >=9 and b < 22, c >= 89 and c
145   * < 102, and d = 1 At the end the table will look something like this: ------------ a| b| c|d|
146   * ------------ 1| 9| 89|1|family:qf|xyz| ----------- 1| 9| 90|1|family:qf|abc| ----------- 1| 9|
147   * 91|1|family:qf|xyz| ------------------------- . ------------------------- .
148   * ------------------------- 1|21|101|1|family:qf|xyz|
149   */
150  private void generateRows(Table ht, byte[] family, byte[] qf) throws IOException {
151    for (int a = 1; a < 2; ++a) {
152      for (int b = 9; b < 22; ++b) {
153        for (int c = 89; c < 102; ++c) {
154          for (int d = 1; d < 2; ++d) {
155            byte[] key = new byte[16];
156            Bytes.putInt(key, 0, a);
157            Bytes.putInt(key, 4, b);
158            Bytes.putInt(key, 8, c);
159            Bytes.putInt(key, 12, d);
160            Put row = new Put(key);
161            if (c % 2 == 0) {
162              row.addColumn(family, qf, Bytes.toBytes("abc"));
163              if (LOG.isInfoEnabled()) {
164                LOG.info("added row: {} with value 'abc'", Arrays.toString(Hex.encodeHex(key)));
165              }
166            } else {
167              row.addColumn(family, qf, Bytes.toBytes("xyz"));
168              if (LOG.isInfoEnabled()) {
169                LOG.info("added row: {} with value 'xyz'", Arrays.toString(Hex.encodeHex(key)));
170              }
171            }
172          }
173        }
174      }
175    }
176    TEST_UTIL.flush();
177  }
178
179  private void setRowFilters(FilterList filterList) {
180    // offset for b as it is second component of "a+b+c+d"
181    // 'a' is at offset 0
182    int bOffset = 4;
183    byte[] b10 = Bytes.toBytes(10); // tests b > 10
184    Filter b10Filter =
185      new RowFilter(CompareOperator.GREATER, new BinaryComponentComparator(b10, bOffset));
186    filterList.addFilter(b10Filter);
187
188    byte[] b20 = Bytes.toBytes(20); // tests b < 20
189    Filter b20Filter =
190      new RowFilter(CompareOperator.LESS, new BinaryComponentComparator(b20, bOffset));
191    filterList.addFilter(b20Filter);
192
193    // offset for c as it is third component of "a+b+c+d"
194    int cOffset = 8;
195    byte[] c90 = Bytes.toBytes(90); // tests c > 90
196    Filter c90Filter =
197      new RowFilter(CompareOperator.GREATER, new BinaryComponentComparator(c90, cOffset));
198    filterList.addFilter(c90Filter);
199
200    byte[] c100 = Bytes.toBytes(100); // tests c < 100
201    Filter c100Filter =
202      new RowFilter(CompareOperator.LESS, new BinaryComponentComparator(c100, cOffset));
203    filterList.addFilter(c100Filter);
204
205    // offset for d as it is fourth component of "a+b+c+d"
206    int dOffset = 12;
207    byte[] d1 = Bytes.toBytes(1); // tests d == 1
208    Filter dFilter =
209      new RowFilter(CompareOperator.EQUAL, new BinaryComponentComparator(d1, dOffset));
210
211    filterList.addFilter(dFilter);
212
213  }
214
215  /**
216   * We have rows with either "abc" or "xyz". We want values which have 'y' at second position of
217   * the string. As a result only values with "xyz" shall be returned
218   */
219  private void setValueFilters(FilterList filterList) {
220    int offset = 1;
221    byte[] y = Bytes.toBytes("y");
222    Filter yFilter =
223      new ValueFilter(CompareOperator.EQUAL, new BinaryComponentComparator(y, offset));
224    filterList.addFilter(yFilter);
225  }
226
227  private Scan createScan(FilterList list) {
228    // build start and end key for scan
229    byte[] startKey = new byte[16]; // key size with four ints
230    Bytes.putInt(startKey, aOffset, 1); // a=1, takes care of a = 1
231    Bytes.putInt(startKey, bOffset, 11); // b=11, takes care of b > 10
232    Bytes.putInt(startKey, cOffset, 91); // c=91,
233    Bytes.putInt(startKey, dOffset, 1); // d=1,
234
235    byte[] endKey = new byte[16];
236    Bytes.putInt(endKey, aOffset, 1); // a=1, takes care of a = 1
237    Bytes.putInt(endKey, bOffset, 20); // b=20, takes care of b < 20
238    Bytes.putInt(endKey, cOffset, 100); // c=100,
239    Bytes.putInt(endKey, dOffset, 1); // d=1,
240
241    // setup scan
242    Scan scan = new Scan().withStartRow(startKey).withStopRow(endKey);
243    scan.setFilter(list);
244    return scan;
245  }
246
247  private List<Cell> getResults(Table ht, Scan scan) throws IOException {
248    ResultScanner scanner = ht.getScanner(scan);
249    List<Cell> results = new ArrayList<>();
250    Result r;
251    while ((r = scanner.next()) != null) {
252      for (Cell kv : r.listCells()) {
253        results.add(kv);
254      }
255    }
256    scanner.close();
257    return results;
258  }
259
260}