001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertNotNull;
022import static org.junit.jupiter.api.Assertions.assertNull;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024
025import java.io.IOException;
026import org.apache.hadoop.fs.Path;
027import org.apache.hadoop.hbase.HBaseTestingUtil;
028import org.apache.hadoop.hbase.TableName;
029import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
030import org.apache.hadoop.hbase.client.Delete;
031import org.apache.hadoop.hbase.client.Get;
032import org.apache.hadoop.hbase.client.Put;
033import org.apache.hadoop.hbase.client.RegionInfo;
034import org.apache.hadoop.hbase.client.RegionInfoBuilder;
035import org.apache.hadoop.hbase.client.Result;
036import org.apache.hadoop.hbase.client.ResultScanner;
037import org.apache.hadoop.hbase.client.Scan;
038import org.apache.hadoop.hbase.client.Scan.ReadType;
039import org.apache.hadoop.hbase.client.TableDescriptor;
040import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
041import org.apache.hadoop.hbase.io.hfile.BlockType;
042import org.apache.hadoop.hbase.regionserver.HRegion.FlushResult;
043import org.apache.hadoop.hbase.testclassification.MediumTests;
044import org.apache.hadoop.hbase.testclassification.RegionServerTests;
045import org.apache.hadoop.hbase.util.Bytes;
046import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
047import org.junit.jupiter.api.AfterAll;
048import org.junit.jupiter.api.AfterEach;
049import org.junit.jupiter.api.BeforeEach;
050import org.junit.jupiter.api.Tag;
051import org.junit.jupiter.api.Test;
052import org.junit.jupiter.api.TestInfo;
053
054/**
055 * A UT to make sure that everything is fine when we fail to load bloom filter.
056 * <p>
057 * See HBASE-27936 for more details.
058 */
059@Tag(RegionServerTests.TAG)
060@Tag(MediumTests.TAG)
061public class TestBloomFilterFaulty {
062
063  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
064
065  private static final byte[] FAMILY = Bytes.toBytes("family");
066
067  private static final byte[] QUAL = Bytes.toBytes("qualifier");
068
069  private static final TableDescriptor TD =
070    TableDescriptorBuilder.newBuilder(TableName.valueOf("test"))
071      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY)
072        .setBloomFilterType(BloomType.ROWPREFIX_FIXED_LENGTH)
073        .setConfiguration("RowPrefixBloomFilter.prefix_length", "2").build())
074      .build();
075
076  private static final RegionInfo RI = RegionInfoBuilder.newBuilder(TD.getTableName()).build();
077
078  @AfterAll
079  public static void tearDownAfterClass() {
080    UTIL.cleanupTestDir();
081  }
082
083  private HRegion region;
084  private String name;
085
086  private void generateHFiles() throws IOException {
087    for (int i = 0; i < 4; i++) {
088      long ts = EnvironmentEdgeManager.currentTime();
089      for (int j = 0; j < 5; j++) {
090        byte[] row = Bytes.toBytes(j);
091        region.put(new Put(row).addColumn(FAMILY, QUAL, ts, Bytes.toBytes(i * 10 + j)));
092        region.delete(new Delete(row).addFamilyVersion(FAMILY, ts));
093      }
094
095      for (int j = 5; j < 10; j++) {
096        byte[] row = Bytes.toBytes(j);
097        region.put(new Put(row).addColumn(FAMILY, QUAL, ts + 1, Bytes.toBytes(i * 10 + j)));
098      }
099
100      FlushResult result = region.flush(true);
101      if (
102        result.getResult() == FlushResult.Result.CANNOT_FLUSH
103          || result.getResult() == FlushResult.Result.CANNOT_FLUSH_MEMSTORE_EMPTY
104      ) {
105        throw new IOException("Can not flush region, flush result: " + result);
106      }
107    }
108  }
109
110  @BeforeEach
111  public void setUp(TestInfo testInfo) throws IOException {
112    this.name = testInfo.getTestMethod().get().getName();
113    Path rootDir = UTIL.getDataTestDir(name);
114    // generate some hfiles so we can have StoreFileReader which has bloomfilters
115    region = HBaseTestingUtil.createRegionAndWAL(RI, rootDir, UTIL.getConfiguration(), TD);
116    generateHFiles();
117    HStore store = region.getStore(FAMILY);
118    for (HStoreFile storefile : store.getStorefiles()) {
119      storefile.initReader();
120      StoreFileReader reader = storefile.getReader();
121      // make sure we load bloom filters correctly
122      assertNotNull(reader.generalBloomFilter);
123      assertNotNull(reader.deleteFamilyBloomFilter);
124    }
125  }
126
127  @AfterEach
128  public void tearDown() throws IOException {
129    if (region != null) {
130      HBaseTestingUtil.closeRegionAndWAL(region);
131    }
132  }
133
134  private void setFaulty(BlockType type) {
135    HStore store = region.getStore(FAMILY);
136    for (HStoreFile storefile : store.getStorefiles()) {
137      storefile.getReader().setBloomFilterFaulty(type);
138    }
139  }
140
141  private void testGet() throws IOException {
142    for (int i = 0; i < 5; i++) {
143      assertTrue(region.get(new Get(Bytes.toBytes(i))).isEmpty());
144    }
145    for (int i = 5; i < 10; i++) {
146      assertEquals(30 + i,
147        Bytes.toInt(region.get(new Get(Bytes.toBytes(i))).getValue(FAMILY, QUAL)));
148    }
149  }
150
151  private void testStreamScan() throws IOException {
152    try (RegionAsTable table = new RegionAsTable(region);
153      ResultScanner scanner = table.getScanner(new Scan().setReadType(ReadType.STREAM))) {
154      for (int i = 5; i < 10; i++) {
155        Result result = scanner.next();
156        assertEquals(i, Bytes.toInt(result.getRow()));
157        assertEquals(30 + i, Bytes.toInt(result.getValue(FAMILY, QUAL)));
158      }
159      assertNull(scanner.next());
160    }
161  }
162
163  private void testRegion() throws IOException {
164    // normal read
165    testGet();
166    // scan with stream reader
167    testStreamScan();
168    // major compact
169    region.compact(true);
170    // test read and scan again
171    testGet();
172    testStreamScan();
173  }
174
175  @Test
176  public void testNoGeneralBloomFilter() throws IOException {
177    setFaulty(BlockType.GENERAL_BLOOM_META);
178    testRegion();
179  }
180
181  @Test
182  public void testNoDeleteFamilyBloomFilter() throws IOException {
183    setFaulty(BlockType.DELETE_FAMILY_BLOOM_META);
184    testRegion();
185  }
186
187  @Test
188  public void testNoAnyBloomFilter() throws IOException {
189    setFaulty(BlockType.GENERAL_BLOOM_META);
190    setFaulty(BlockType.DELETE_FAMILY_BLOOM_META);
191    testRegion();
192  }
193}