001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.jupiter.api.Assertions.assertTrue;
021import static org.junit.jupiter.api.Assertions.fail;
022
023import java.nio.charset.Charset;
024import java.util.ArrayList;
025import java.util.Collections;
026import java.util.LinkedList;
027import java.util.List;
028import java.util.stream.Stream;
029import org.apache.commons.io.IOUtils;
030import org.apache.commons.text.StringSubstitutor;
031import org.apache.hadoop.conf.Configuration;
032import org.apache.hadoop.hbase.CompareOperator;
033import org.apache.hadoop.hbase.DoNotRetryIOException;
034import org.apache.hadoop.hbase.HBaseConfiguration;
035import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate;
036import org.apache.hadoop.hbase.HBaseTestingUtil;
037import org.apache.hadoop.hbase.filter.MultiRowRangeFilter.RowRange;
038import org.apache.hadoop.hbase.testclassification.FilterTests;
039import org.apache.hadoop.hbase.testclassification.MediumTests;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.apache.hadoop.hbase.util.ClassLoaderTestHelper;
042import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
043import org.apache.hadoop.hbase.util.Pair;
044import org.junit.jupiter.api.AfterAll;
045import org.junit.jupiter.api.Tag;
046import org.junit.jupiter.api.TestTemplate;
047import org.junit.jupiter.params.provider.Arguments;
048
049import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
050import org.apache.hadoop.hbase.shaded.protobuf.generated.FilterProtos;
051
052@Tag(FilterTests.TAG)
053@Tag(MediumTests.TAG)
054@HBaseParameterizedTestTemplate(name = "{index}: allowFastReflectionFallthrough={0}")
055public class TestFilterSerialization {
056
057  public boolean allowFastReflectionFallthrough;
058
059  public TestFilterSerialization(boolean allowFastReflectionFallthrough) {
060    this.allowFastReflectionFallthrough = allowFastReflectionFallthrough;
061  }
062
063  public static Stream<Arguments> parameters() {
064    return Stream.of(Arguments.of(true), Arguments.of(false));
065  }
066
067  @AfterAll
068  public static void afterClass() throws Exception {
069    // set back to true so that it doesn't affect any other tests
070    ProtobufUtil.setAllowFastReflectionFallthrough(true);
071  }
072
073  @TestTemplate
074  public void testColumnCountGetFilter() throws Exception {
075    ColumnCountGetFilter columnCountGetFilter = new ColumnCountGetFilter(1);
076    assertTrue(columnCountGetFilter.areSerializedFieldsEqual(
077      ProtobufUtil.toFilter(ProtobufUtil.toFilter(columnCountGetFilter))));
078  }
079
080  @TestTemplate
081  public void testColumnPaginationFilter() throws Exception {
082    ColumnPaginationFilter columnPaginationFilter = new ColumnPaginationFilter(1, 7);
083    assertTrue(columnPaginationFilter.areSerializedFieldsEqual(
084      ProtobufUtil.toFilter(ProtobufUtil.toFilter(columnPaginationFilter))));
085  }
086
087  @TestTemplate
088  public void testColumnPrefixFilter() throws Exception {
089    // empty string
090    ColumnPrefixFilter columnPrefixFilter = new ColumnPrefixFilter(Bytes.toBytes(""));
091    assertTrue(columnPrefixFilter
092      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(columnPrefixFilter))));
093
094    // non-empty string
095    columnPrefixFilter = new ColumnPrefixFilter(Bytes.toBytes(""));
096    assertTrue(columnPrefixFilter
097      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(columnPrefixFilter))));
098  }
099
100  @TestTemplate
101  public void testColumnRangeFilter() throws Exception {
102    // null columns
103    ColumnRangeFilter columnRangeFilter = new ColumnRangeFilter(null, true, null, false);
104    assertTrue(columnRangeFilter
105      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(columnRangeFilter))));
106
107    // non-null columns
108    columnRangeFilter = new ColumnRangeFilter(Bytes.toBytes("a"), false, Bytes.toBytes("b"), true);
109    assertTrue(columnRangeFilter
110      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(columnRangeFilter))));
111  }
112
113  @TestTemplate
114  public void testDependentColumnFilter() throws Exception {
115    // null column qualifier/family
116    DependentColumnFilter dependentColumnFilter = new DependentColumnFilter(null, null);
117    assertTrue(dependentColumnFilter.areSerializedFieldsEqual(
118      ProtobufUtil.toFilter(ProtobufUtil.toFilter(dependentColumnFilter))));
119
120    // non-null column qualifier/family
121    dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes("family"),
122      Bytes.toBytes("qual"), true, CompareOperator.GREATER_OR_EQUAL,
123      new BitComparator(Bytes.toBytes("bitComparator"), BitComparator.BitwiseOp.OR));
124    assertTrue(dependentColumnFilter.areSerializedFieldsEqual(
125      ProtobufUtil.toFilter(ProtobufUtil.toFilter(dependentColumnFilter))));
126  }
127
128  @TestTemplate
129  public void testFamilyFilter() throws Exception {
130    FamilyFilter familyFilter = new FamilyFilter(CompareOperator.EQUAL,
131      new BinaryPrefixComparator(Bytes.toBytes("testValueOne")));
132    assertTrue(familyFilter
133      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(familyFilter))));
134  }
135
136  @TestTemplate
137  public void testFilterList() throws Exception {
138    // empty filter list
139    FilterList filterList = new FilterList(new LinkedList<>());
140    assertTrue(filterList
141      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(filterList))));
142
143    // non-empty filter list
144    LinkedList<Filter> list = new LinkedList<>();
145    list.add(new ColumnCountGetFilter(1));
146    list.add(new RowFilter(CompareOperator.EQUAL, new SubstringComparator("testFilterList")));
147    assertTrue(filterList
148      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(filterList))));
149  }
150
151  @TestTemplate
152  public void testFilterWrapper() throws Exception {
153    FilterWrapper filterWrapper =
154      new FilterWrapper(new ColumnRangeFilter(Bytes.toBytes("e"), false, Bytes.toBytes("f"), true));
155    assertTrue(filterWrapper
156      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(filterWrapper))));
157  }
158
159  @TestTemplate
160  public void testFirstKeyOnlyFilter() throws Exception {
161    FirstKeyOnlyFilter firstKeyOnlyFilter = new FirstKeyOnlyFilter();
162    assertTrue(firstKeyOnlyFilter
163      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(firstKeyOnlyFilter))));
164  }
165
166  @TestTemplate
167  public void testFuzzyRowFilter() throws Exception {
168    LinkedList<Pair<byte[], byte[]>> fuzzyList = new LinkedList<>();
169    fuzzyList.add(new Pair<>(Bytes.toBytes("999"), new byte[] { 0, 0, 1 }));
170    fuzzyList.add(new Pair<>(Bytes.toBytes("abcd"), new byte[] { 1, 0, 1, 1 }));
171    FuzzyRowFilter fuzzyRowFilter = new FuzzyRowFilter(fuzzyList);
172    assertTrue(fuzzyRowFilter
173      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(fuzzyRowFilter))));
174  }
175
176  @TestTemplate
177  public void testInclusiveStopFilter() throws Exception {
178    // InclusveStopFilter with null stopRowKey
179    InclusiveStopFilter inclusiveStopFilter = new InclusiveStopFilter(null);
180    assertTrue(inclusiveStopFilter
181      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(inclusiveStopFilter))));
182
183    // InclusveStopFilter with non-null stopRowKey
184    inclusiveStopFilter = new InclusiveStopFilter(Bytes.toBytes("inclusiveStopFilter"));
185    assertTrue(inclusiveStopFilter
186      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(inclusiveStopFilter))));
187  }
188
189  @TestTemplate
190  public void testKeyOnlyFilter() throws Exception {
191    // KeyOnlyFilter with lenAsVal
192    KeyOnlyFilter keyOnlyFilter = new KeyOnlyFilter(true);
193    assertTrue(keyOnlyFilter
194      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(keyOnlyFilter))));
195
196    // KeyOnlyFilter without lenAsVal
197    keyOnlyFilter = new KeyOnlyFilter();
198    assertTrue(keyOnlyFilter
199      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(keyOnlyFilter))));
200  }
201
202  @TestTemplate
203  public void testMultipleColumnPrefixFilter() throws Exception {
204    // empty array
205    byte[][] prefixes = null;
206    MultipleColumnPrefixFilter multipleColumnPrefixFilter =
207      new MultipleColumnPrefixFilter(prefixes);
208    assertTrue(multipleColumnPrefixFilter.areSerializedFieldsEqual(
209      ProtobufUtil.toFilter(ProtobufUtil.toFilter(multipleColumnPrefixFilter))));
210
211    // non-empty array
212    prefixes = new byte[2][];
213    prefixes[0] = Bytes.toBytes("a");
214    prefixes[1] = Bytes.toBytes("");
215    multipleColumnPrefixFilter = new MultipleColumnPrefixFilter(prefixes);
216    assertTrue(multipleColumnPrefixFilter.areSerializedFieldsEqual(
217      ProtobufUtil.toFilter(ProtobufUtil.toFilter(multipleColumnPrefixFilter))));
218  }
219
220  @TestTemplate
221  public void testPageFilter() throws Exception {
222    PageFilter pageFilter = new PageFilter(6);
223    assertTrue(pageFilter
224      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(pageFilter))));
225  }
226
227  @TestTemplate
228  public void testPrefixFilter() throws Exception {
229    // null prefix
230    PrefixFilter prefixFilter = new PrefixFilter(null);
231    assertTrue(prefixFilter
232      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(prefixFilter))));
233
234    // non-null prefix
235    prefixFilter = new PrefixFilter(Bytes.toBytes("abc"));
236    assertTrue(prefixFilter
237      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(prefixFilter))));
238  }
239
240  @TestTemplate
241  public void testQualifierFilter() throws Exception {
242    QualifierFilter qualifierFilter =
243      new QualifierFilter(CompareOperator.EQUAL, new NullComparator());
244    assertTrue(qualifierFilter
245      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(qualifierFilter))));
246  }
247
248  @TestTemplate
249  public void testRandomRowFilter() throws Exception {
250    RandomRowFilter randomRowFilter = new RandomRowFilter((float) 0.1);
251    assertTrue(randomRowFilter
252      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(randomRowFilter))));
253  }
254
255  @TestTemplate
256  public void testRowFilter() throws Exception {
257    RowFilter rowFilter =
258      new RowFilter(CompareOperator.EQUAL, new SubstringComparator("testRowFilter"));
259    assertTrue(
260      rowFilter.areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(rowFilter))));
261  }
262
263  @TestTemplate
264  public void testSingleColumnValueExcludeFilter() throws Exception {
265    // null family/column SingleColumnValueExcludeFilter
266    SingleColumnValueExcludeFilter singleColumnValueExcludeFilter =
267      new SingleColumnValueExcludeFilter(null, null, CompareOperator.GREATER_OR_EQUAL,
268        Bytes.toBytes("value"));
269    assertTrue(singleColumnValueExcludeFilter.areSerializedFieldsEqual(
270      ProtobufUtil.toFilter(ProtobufUtil.toFilter(singleColumnValueExcludeFilter))));
271
272    // non-null family/column SingleColumnValueFilter
273    singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes("fam"),
274      Bytes.toBytes("qual"), CompareOperator.LESS_OR_EQUAL, new NullComparator(), false, false);
275    assertTrue(singleColumnValueExcludeFilter.areSerializedFieldsEqual(
276      ProtobufUtil.toFilter(ProtobufUtil.toFilter(singleColumnValueExcludeFilter))));
277  }
278
279  @TestTemplate
280  public void testSingleColumnValueFilter() throws Exception {
281    // null family/column SingleColumnValueFilter
282    SingleColumnValueFilter singleColumnValueFilter =
283      new SingleColumnValueFilter(null, null, CompareOperator.LESS, Bytes.toBytes("value"));
284    assertTrue(singleColumnValueFilter.areSerializedFieldsEqual(
285      ProtobufUtil.toFilter(ProtobufUtil.toFilter(singleColumnValueFilter))));
286
287    // non-null family/column SingleColumnValueFilter
288    singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes("family"),
289      Bytes.toBytes("qualifier"), CompareOperator.NOT_EQUAL, new NullComparator(), true, true);
290    assertTrue(singleColumnValueFilter.areSerializedFieldsEqual(
291      ProtobufUtil.toFilter(ProtobufUtil.toFilter(singleColumnValueFilter))));
292  }
293
294  @TestTemplate
295  public void testSkipFilter() throws Exception {
296    SkipFilter skipFilter = new SkipFilter(new PageFilter(6));
297    assertTrue(skipFilter
298      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(skipFilter))));
299  }
300
301  @TestTemplate
302  public void testTimestampsFilter() throws Exception {
303    // Empty timestamp list
304    TimestampsFilter timestampsFilter = new TimestampsFilter(new LinkedList<>());
305    assertTrue(timestampsFilter
306      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(timestampsFilter))));
307
308    // Non-empty timestamp list
309    LinkedList<Long> list = new LinkedList<>();
310    list.add(EnvironmentEdgeManager.currentTime());
311    list.add(EnvironmentEdgeManager.currentTime());
312    timestampsFilter = new TimestampsFilter(list);
313    assertTrue(timestampsFilter
314      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(timestampsFilter))));
315  }
316
317  @TestTemplate
318  public void testValueFilter() throws Exception {
319    ValueFilter valueFilter =
320      new ValueFilter(CompareOperator.NO_OP, new BinaryComparator(Bytes.toBytes("testValueOne")));
321    assertTrue(valueFilter
322      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(valueFilter))));
323  }
324
325  @TestTemplate
326  public void testWhileMatchFilter() throws Exception {
327    WhileMatchFilter whileMatchFilter = new WhileMatchFilter(
328      new ColumnRangeFilter(Bytes.toBytes("c"), false, Bytes.toBytes("d"), true));
329    assertTrue(whileMatchFilter
330      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(whileMatchFilter))));
331  }
332
333  @TestTemplate
334  public void testMultiRowRangeFilter() throws Exception {
335    List<RowRange> ranges = new ArrayList<>();
336    ranges.add(new RowRange(Bytes.toBytes(30), true, Bytes.toBytes(40), false));
337    ranges.add(new RowRange(Bytes.toBytes(10), true, Bytes.toBytes(20), false));
338    ranges.add(new RowRange(Bytes.toBytes(60), true, Bytes.toBytes(70), false));
339
340    MultiRowRangeFilter multiRowRangeFilter = new MultiRowRangeFilter(ranges);
341    assertTrue(multiRowRangeFilter
342      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(multiRowRangeFilter))));
343  }
344
345  @TestTemplate
346  public void testColumnValueFilter() throws Exception {
347    ColumnValueFilter columnValueFilter = new ColumnValueFilter(Bytes.toBytes("family"),
348      Bytes.toBytes("qualifier"), CompareOperator.EQUAL, Bytes.toBytes("value"));
349    assertTrue(columnValueFilter
350      .areSerializedFieldsEqual(ProtobufUtil.toFilter(ProtobufUtil.toFilter(columnValueFilter))));
351  }
352
353  /**
354   * Test that we can load and deserialize custom filters. Good to have generally, but also proves
355   * that this still works after HBASE-27276 despite not going through our fast function caches.
356   */
357  @TestTemplate
358  public void testCustomFilter() throws Exception {
359    Filter baseFilter = new PrefixFilter("foo".getBytes());
360    FilterProtos.Filter filterProto = ProtobufUtil.toFilter(baseFilter);
361    String suffix = "" + System.currentTimeMillis() + allowFastReflectionFallthrough;
362    String className = "CustomLoadedFilter" + suffix;
363    filterProto = filterProto.toBuilder().setName(className).build();
364
365    Configuration conf = HBaseConfiguration.create();
366    HBaseTestingUtil testUtil = new HBaseTestingUtil();
367    String dataTestDir = testUtil.getDataTestDir().toString();
368
369    // First make sure the test bed is clean, delete any pre-existing class.
370    // Below toComparator call is expected to fail because the comparator is not loaded now
371    ClassLoaderTestHelper.deleteClass(className, dataTestDir, conf);
372    try {
373      Filter filter = ProtobufUtil.toFilter(filterProto);
374      fail("expected to fail");
375    } catch (DoNotRetryIOException e) {
376      // do nothing, this is expected
377    }
378
379    // Write a jar to be loaded into the classloader
380    String code = StringSubstitutor
381      .replace(IOUtils.toString(getClass().getResourceAsStream("/CustomLoadedFilter.java.template"),
382        Charset.defaultCharset()), Collections.singletonMap("suffix", suffix));
383    ClassLoaderTestHelper.buildJar(dataTestDir, className, code,
384      ClassLoaderTestHelper.localDirPath(conf));
385
386    // Disallow fallthrough at first. We expect below to fail because the custom filter is not
387    // available at initialization so not in the cache.
388    ProtobufUtil.setAllowFastReflectionFallthrough(false);
389    try {
390      ProtobufUtil.toFilter(filterProto);
391      fail("expected to fail");
392    } catch (DoNotRetryIOException e) {
393      // do nothing, this is expected
394    }
395
396    // Now the deserialization should pass with fallthrough enabled. This proves that custom
397    // filters can work despite not being supported by cache.
398    ProtobufUtil.setAllowFastReflectionFallthrough(true);
399    ProtobufUtil.toFilter(filterProto);
400
401  }
402
403}