001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.jupiter.api.Assertions.assertNull;
021import static org.junit.jupiter.api.Assertions.assertTrue;
022import static org.junit.jupiter.api.Assertions.fail;
023
024import java.io.IOException;
025import java.math.BigDecimal;
026import java.nio.charset.Charset;
027import java.util.Collections;
028import java.util.regex.Pattern;
029import java.util.stream.Stream;
030import org.apache.commons.io.IOUtils;
031import org.apache.commons.text.StringSubstitutor;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.hbase.HBaseConfiguration;
034import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate;
035import org.apache.hadoop.hbase.HBaseTestingUtil;
036import org.apache.hadoop.hbase.testclassification.FilterTests;
037import org.apache.hadoop.hbase.testclassification.SmallTests;
038import org.apache.hadoop.hbase.util.Bytes;
039import org.apache.hadoop.hbase.util.ClassLoaderTestHelper;
040import org.junit.jupiter.api.AfterAll;
041import org.junit.jupiter.api.Tag;
042import org.junit.jupiter.api.TestTemplate;
043import org.junit.jupiter.params.provider.Arguments;
044
045import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
046import org.apache.hadoop.hbase.shaded.protobuf.generated.ComparatorProtos;
047
048@Tag(FilterTests.TAG)
049@Tag(SmallTests.TAG)
050@HBaseParameterizedTestTemplate(name = "{index}: allowFastReflectionFallthrough={0}")
051public class TestComparatorSerialization {
052
053  public boolean allowFastReflectionFallthrough;
054
055  public TestComparatorSerialization(boolean allowFastReflectionFallthrough) {
056    this.allowFastReflectionFallthrough = allowFastReflectionFallthrough;
057  }
058
059  public static Stream<Arguments> parameters() {
060    return Stream.of(Arguments.of(true), Arguments.of(false));
061  }
062
063  @AfterAll
064  public static void afterClass() throws Exception {
065    // set back to true so that it doesn't affect any other tests
066    ProtobufUtil.setAllowFastReflectionFallthrough(true);
067  }
068
069  @TestTemplate
070  public void testBinaryComparator() throws Exception {
071    BinaryComparator binaryComparator = new BinaryComparator(Bytes.toBytes("binaryComparator"));
072    assertTrue(binaryComparator.areSerializedFieldsEqual(
073      ProtobufUtil.toComparator(ProtobufUtil.toComparator(binaryComparator))));
074  }
075
076  @TestTemplate
077  public void testBinaryPrefixComparator() throws Exception {
078    BinaryPrefixComparator binaryPrefixComparator =
079      new BinaryPrefixComparator(Bytes.toBytes("binaryPrefixComparator"));
080    assertTrue(binaryPrefixComparator.areSerializedFieldsEqual(
081      ProtobufUtil.toComparator(ProtobufUtil.toComparator(binaryPrefixComparator))));
082  }
083
084  @TestTemplate
085  public void testBitComparator() throws Exception {
086    BitComparator bitComparator =
087      new BitComparator(Bytes.toBytes("bitComparator"), BitComparator.BitwiseOp.XOR);
088    assertTrue(bitComparator.areSerializedFieldsEqual(
089      ProtobufUtil.toComparator(ProtobufUtil.toComparator(bitComparator))));
090  }
091
092  @TestTemplate
093  public void testNullComparator() throws Exception {
094    NullComparator nullComparator = new NullComparator();
095    assertTrue(nullComparator.areSerializedFieldsEqual(
096      ProtobufUtil.toComparator(ProtobufUtil.toComparator(nullComparator))));
097  }
098
099  @TestTemplate
100  public void testRegexStringComparator() throws Exception {
101    // test without specifying flags
102    RegexStringComparator regexStringComparator = new RegexStringComparator(".+-2");
103    assertTrue(regexStringComparator.areSerializedFieldsEqual(
104      ProtobufUtil.toComparator(ProtobufUtil.toComparator(regexStringComparator))));
105
106    // test with specifying flags
107    try {
108      new RegexStringComparator("regex", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
109    } catch (Throwable t) {
110      assertNull(t, "Exception occurred while created the RegexStringComparator object");
111    }
112  }
113
114  @TestTemplate
115  public void testSubstringComparator() throws Exception {
116    SubstringComparator substringComparator = new SubstringComparator("substr");
117    assertTrue(substringComparator.areSerializedFieldsEqual(
118      ProtobufUtil.toComparator(ProtobufUtil.toComparator(substringComparator))));
119  }
120
121  @TestTemplate
122  public void testBigDecimalComparator() throws Exception {
123    BigDecimal bigDecimal = new BigDecimal(Double.MIN_VALUE);
124    BigDecimalComparator bigDecimalComparator = new BigDecimalComparator(bigDecimal);
125    assertTrue(bigDecimalComparator.areSerializedFieldsEqual(
126      ProtobufUtil.toComparator(ProtobufUtil.toComparator(bigDecimalComparator))));
127  }
128
129  /**
130   * Test that we can load and deserialize custom comparators. Good to have generally, but also
131   * proves that this still works after HBASE-27276 despite not going through our fast function
132   * caches.
133   */
134  @TestTemplate
135  public void testCustomComparator() throws Exception {
136    ByteArrayComparable baseFilter = new BinaryComparator("foo".getBytes());
137    ComparatorProtos.Comparator proto = ProtobufUtil.toComparator(baseFilter);
138    String suffix = "" + System.currentTimeMillis() + allowFastReflectionFallthrough;
139    String className = "CustomLoadedComparator" + suffix;
140    proto = proto.toBuilder().setName(className).build();
141
142    Configuration conf = HBaseConfiguration.create();
143    HBaseTestingUtil testUtil = new HBaseTestingUtil();
144    String dataTestDir = testUtil.getDataTestDir().toString();
145
146    // First make sure the test bed is clean, delete any pre-existing class.
147    // Below toComparator call is expected to fail because the comparator is not loaded now
148    ClassLoaderTestHelper.deleteClass(className, dataTestDir, conf);
149    try {
150      ProtobufUtil.toComparator(proto);
151      fail("expected to fail");
152    } catch (IOException e) {
153      // do nothing, this is expected
154    }
155
156    // Write a jar to be loaded into the classloader
157    String code = StringSubstitutor.replace(
158      IOUtils.toString(getClass().getResourceAsStream("/CustomLoadedComparator.java.template"),
159        Charset.defaultCharset()),
160      Collections.singletonMap("suffix", suffix));
161    ClassLoaderTestHelper.buildJar(dataTestDir, className, code,
162      ClassLoaderTestHelper.localDirPath(conf));
163
164    // Disallow fallthrough at first. We expect below to fail because the custom comparator is not
165    // available at initialization so not in the cache.
166    ProtobufUtil.setAllowFastReflectionFallthrough(false);
167    try {
168      ProtobufUtil.toComparator(proto);
169      fail("expected to fail");
170    } catch (IOException e) {
171      // do nothing, this is expected
172    }
173
174    // Now the deserialization should pass with fallthrough enabled. This proves that custom
175    // comparators can work despite not being supported by cache.
176    ProtobufUtil.setAllowFastReflectionFallthrough(true);
177    ProtobufUtil.toComparator(proto);
178  }
179
180}