Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.HashMap;
025import java.util.HashSet;
026import java.util.List;
027import java.util.Map;
028import java.util.Set;
029import org.apache.hadoop.hbase.Cell;
030import org.apache.hadoop.hbase.HBaseTestingUtil;
031import org.apache.hadoop.hbase.KeyValue;
032import org.apache.hadoop.hbase.KeyValueTestUtil;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
036import org.apache.hadoop.hbase.client.Durability;
037import org.apache.hadoop.hbase.client.Put;
038import org.apache.hadoop.hbase.client.RegionInfo;
039import org.apache.hadoop.hbase.client.RegionInfoBuilder;
040import org.apache.hadoop.hbase.client.Scan;
041import org.apache.hadoop.hbase.client.TableDescriptor;
042import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
043import org.apache.hadoop.hbase.regionserver.HRegion;
044import org.apache.hadoop.hbase.regionserver.InternalScanner;
045import org.apache.hadoop.hbase.testclassification.FilterTests;
046import org.apache.hadoop.hbase.testclassification.MediumTests;
047import org.apache.hadoop.hbase.util.Bytes;
048import org.junit.jupiter.api.Tag;
049import org.junit.jupiter.api.Test;
050import org.junit.jupiter.api.TestInfo;
051
052@Tag(FilterTests.TAG)
053@Tag(MediumTests.TAG)
054public class TestMultipleColumnPrefixFilter {
055
056  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
057
058  @Test
059  public void testMultipleColumnPrefixFilter(TestInfo testInfo) throws IOException {
060    String family = "Family";
061    TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder
062      .newBuilder(TableName.valueOf(testInfo.getTestMethod().get().getName()));
063    ColumnFamilyDescriptor columnFamilyDescriptor =
064      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)).setMaxVersions(3).build();
065    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
066    TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
067    // HRegionInfo info = new HRegionInfo(htd, null, null, false);
068    RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
069    HRegion region = HBaseTestingUtil.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
070      TEST_UTIL.getConfiguration(), tableDescriptor);
071
072    List<String> rows = generateRandomWords(100, "row");
073    List<String> columns = generateRandomWords(10000, "column");
074    long maxTimestamp = 2;
075
076    List<Cell> kvList = new ArrayList<>();
077
078    Map<String, List<Cell>> prefixMap = new HashMap<>();
079
080    prefixMap.put("p", new ArrayList<>());
081    prefixMap.put("q", new ArrayList<>());
082    prefixMap.put("s", new ArrayList<>());
083
084    String valueString = "ValueString";
085
086    for (String row : rows) {
087      Put p = new Put(Bytes.toBytes(row));
088      p.setDurability(Durability.SKIP_WAL);
089      for (String column : columns) {
090        for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
091          KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, valueString);
092          p.add(kv);
093          kvList.add(kv);
094          for (String s : prefixMap.keySet()) {
095            if (column.startsWith(s)) {
096              prefixMap.get(s).add(kv);
097            }
098          }
099        }
100      }
101      region.put(p);
102    }
103
104    MultipleColumnPrefixFilter filter;
105    Scan scan = new Scan();
106    scan.readAllVersions();
107    byte[][] filter_prefix = new byte[2][];
108    filter_prefix[0] = new byte[] { 'p' };
109    filter_prefix[1] = new byte[] { 'q' };
110
111    filter = new MultipleColumnPrefixFilter(filter_prefix);
112    scan.setFilter(filter);
113    List<Cell> results = new ArrayList<>();
114    InternalScanner scanner = region.getScanner(scan);
115    while (scanner.next(results))
116      ;
117    assertEquals(prefixMap.get("p").size() + prefixMap.get("q").size(), results.size());
118
119    HBaseTestingUtil.closeRegionAndWAL(region);
120  }
121
122  @Test
123  public void testMultipleColumnPrefixFilterWithManyFamilies(TestInfo testInfo) throws IOException {
124    String family1 = "Family1";
125    String family2 = "Family2";
126    TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder
127      .newBuilder(TableName.valueOf(testInfo.getTestMethod().get().getName()));
128    ColumnFamilyDescriptor columnFamilyDescriptor =
129      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family1)).setMaxVersions(3).build();
130    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
131    columnFamilyDescriptor =
132      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family2)).setMaxVersions(3).build();
133    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
134    TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
135    RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
136    HRegion region = HBaseTestingUtil.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
137      TEST_UTIL.getConfiguration(), tableDescriptor);
138
139    List<String> rows = generateRandomWords(100, "row");
140    List<String> columns = generateRandomWords(10000, "column");
141    long maxTimestamp = 3;
142
143    List<Cell> kvList = new ArrayList<>();
144
145    Map<String, List<Cell>> prefixMap = new HashMap<>();
146
147    prefixMap.put("p", new ArrayList<>());
148    prefixMap.put("q", new ArrayList<>());
149    prefixMap.put("s", new ArrayList<>());
150
151    String valueString = "ValueString";
152
153    for (String row : rows) {
154      Put p = new Put(Bytes.toBytes(row));
155      p.setDurability(Durability.SKIP_WAL);
156      for (String column : columns) {
157        for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
158          double rand = Math.random();
159          Cell kv;
160          if (rand < 0.5) {
161            kv = KeyValueTestUtil.create(row, family1, column, timestamp, valueString);
162          } else {
163            kv = KeyValueTestUtil.create(row, family2, column, timestamp, valueString);
164          }
165          p.add(kv);
166          kvList.add(kv);
167          for (String s : prefixMap.keySet()) {
168            if (column.startsWith(s)) {
169              prefixMap.get(s).add(kv);
170            }
171          }
172        }
173      }
174      region.put(p);
175    }
176
177    MultipleColumnPrefixFilter filter;
178    Scan scan = new Scan();
179    scan.readAllVersions();
180    byte[][] filter_prefix = new byte[2][];
181    filter_prefix[0] = new byte[] { 'p' };
182    filter_prefix[1] = new byte[] { 'q' };
183
184    filter = new MultipleColumnPrefixFilter(filter_prefix);
185    scan.setFilter(filter);
186    List<Cell> results = new ArrayList<>();
187    InternalScanner scanner = region.getScanner(scan);
188    while (scanner.next(results))
189      ;
190    assertEquals(prefixMap.get("p").size() + prefixMap.get("q").size(), results.size());
191
192    HBaseTestingUtil.closeRegionAndWAL(region);
193  }
194
195  @Test
196  public void testMultipleColumnPrefixFilterWithColumnPrefixFilter(TestInfo testInfo)
197    throws IOException {
198    String family = "Family";
199    TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder
200      .newBuilder(TableName.valueOf(testInfo.getTestMethod().get().getName()));
201    ColumnFamilyDescriptor columnFamilyDescriptor =
202      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)).build();
203    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
204    TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
205    RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
206    HRegion region = HBaseTestingUtil.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
207      TEST_UTIL.getConfiguration(), tableDescriptor);
208
209    List<String> rows = generateRandomWords(100, "row");
210    List<String> columns = generateRandomWords(10000, "column");
211    long maxTimestamp = 2;
212
213    String valueString = "ValueString";
214
215    for (String row : rows) {
216      Put p = new Put(Bytes.toBytes(row));
217      p.setDurability(Durability.SKIP_WAL);
218      for (String column : columns) {
219        for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
220          KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, valueString);
221          p.add(kv);
222        }
223      }
224      region.put(p);
225    }
226
227    MultipleColumnPrefixFilter multiplePrefixFilter;
228    Scan scan1 = new Scan();
229    scan1.readAllVersions();
230    byte[][] filter_prefix = new byte[1][];
231    filter_prefix[0] = new byte[] { 'p' };
232
233    multiplePrefixFilter = new MultipleColumnPrefixFilter(filter_prefix);
234    scan1.setFilter(multiplePrefixFilter);
235    List<Cell> results1 = new ArrayList<>();
236    InternalScanner scanner1 = region.getScanner(scan1);
237    while (scanner1.next(results1))
238      ;
239
240    ColumnPrefixFilter singlePrefixFilter;
241    Scan scan2 = new Scan();
242    scan2.readAllVersions();
243    singlePrefixFilter = new ColumnPrefixFilter(Bytes.toBytes("p"));
244
245    scan2.setFilter(singlePrefixFilter);
246    List<Cell> results2 = new ArrayList<>();
247    InternalScanner scanner2 = region.getScanner(scan1);
248    while (scanner2.next(results2))
249      ;
250
251    assertEquals(results1.size(), results2.size());
252
253    HBaseTestingUtil.closeRegionAndWAL(region);
254  }
255
256  List<String> generateRandomWords(int numberOfWords, String suffix) {
257    Set<String> wordSet = new HashSet<>();
258    for (int i = 0; i < numberOfWords; i++) {
259      int lengthOfWords = (int) (Math.random() * 2) + 1;
260      char[] wordChar = new char[lengthOfWords];
261      for (int j = 0; j < wordChar.length; j++) {
262        wordChar[j] = (char) (Math.random() * 26 + 97);
263      }
264      String word;
265      if (suffix == null) {
266        word = new String(wordChar);
267      } else {
268        word = new String(wordChar) + suffix;
269      }
270      wordSet.add(word);
271    }
272    List<String> wordList = new ArrayList<>(wordSet);
273    return wordList;
274  }
275
276}