001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.rest; 019 020import static org.junit.jupiter.api.Assertions.assertEquals; 021import static org.junit.jupiter.api.Assertions.assertNotNull; 022import static org.junit.jupiter.api.Assertions.assertTrue; 023 024import java.io.ByteArrayInputStream; 025import java.io.StringWriter; 026import java.util.ArrayList; 027import java.util.Arrays; 028import java.util.Iterator; 029import java.util.List; 030import javax.xml.bind.JAXBContext; 031import javax.xml.bind.Marshaller; 032import javax.xml.bind.Unmarshaller; 033import org.apache.hadoop.hbase.CellUtil; 034import org.apache.hadoop.hbase.CompareOperator; 035import org.apache.hadoop.hbase.HBaseTestingUtil; 036import org.apache.hadoop.hbase.HConstants; 037import org.apache.hadoop.hbase.KeyValue; 038import org.apache.hadoop.hbase.TableName; 039import org.apache.hadoop.hbase.client.Admin; 040import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 041import org.apache.hadoop.hbase.client.Delete; 042import org.apache.hadoop.hbase.client.Durability; 043import org.apache.hadoop.hbase.client.Put; 044import org.apache.hadoop.hbase.client.Scan; 045import org.apache.hadoop.hbase.client.Table; 046import org.apache.hadoop.hbase.client.TableDescriptor; 047import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 048import org.apache.hadoop.hbase.filter.BinaryComparator; 049import org.apache.hadoop.hbase.filter.Filter; 050import org.apache.hadoop.hbase.filter.FilterList; 051import org.apache.hadoop.hbase.filter.FilterList.Operator; 052import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; 053import org.apache.hadoop.hbase.filter.FuzzyRowFilter; 054import org.apache.hadoop.hbase.filter.InclusiveStopFilter; 055import org.apache.hadoop.hbase.filter.MultiRowRangeFilter; 056import org.apache.hadoop.hbase.filter.PageFilter; 057import org.apache.hadoop.hbase.filter.PrefixFilter; 058import org.apache.hadoop.hbase.filter.QualifierFilter; 059import org.apache.hadoop.hbase.filter.RegexStringComparator; 060import org.apache.hadoop.hbase.filter.RowFilter; 061import org.apache.hadoop.hbase.filter.SkipFilter; 062import org.apache.hadoop.hbase.filter.SubstringComparator; 063import org.apache.hadoop.hbase.filter.ValueFilter; 064import org.apache.hadoop.hbase.rest.client.Client; 065import org.apache.hadoop.hbase.rest.client.Cluster; 066import org.apache.hadoop.hbase.rest.client.Response; 067import org.apache.hadoop.hbase.rest.model.CellModel; 068import org.apache.hadoop.hbase.rest.model.CellSetModel; 069import org.apache.hadoop.hbase.rest.model.RowModel; 070import org.apache.hadoop.hbase.rest.model.ScannerModel; 071import org.apache.hadoop.hbase.testclassification.MediumTests; 072import org.apache.hadoop.hbase.testclassification.RestTests; 073import org.apache.hadoop.hbase.util.Bytes; 074import org.apache.hadoop.hbase.util.Pair; 075import org.junit.jupiter.api.AfterAll; 076import org.junit.jupiter.api.BeforeAll; 077import org.junit.jupiter.api.Tag; 078import org.junit.jupiter.api.Test; 079import org.slf4j.Logger; 080import org.slf4j.LoggerFactory; 081 082@Tag(RestTests.TAG) 083@Tag(MediumTests.TAG) 084public class TestScannersWithFilters { 085 086 private static final Logger LOG = LoggerFactory.getLogger(TestScannersWithFilters.class); 087 088 private static final TableName TABLE = TableName.valueOf("TestScannersWithFilters"); 089 090 private static final byte[][] ROWS_ONE = { Bytes.toBytes("testRowOne-0"), 091 Bytes.toBytes("testRowOne-1"), Bytes.toBytes("testRowOne-2"), Bytes.toBytes("testRowOne-3") }; 092 093 private static final byte[][] ROWS_TWO = { Bytes.toBytes("testRowTwo-0"), 094 Bytes.toBytes("testRowTwo-1"), Bytes.toBytes("testRowTwo-2"), Bytes.toBytes("testRowTwo-3") }; 095 096 private static final byte[][] FAMILIES = 097 { Bytes.toBytes("testFamilyOne"), Bytes.toBytes("testFamilyTwo") }; 098 099 private static final byte[][] QUALIFIERS_ONE = 100 { Bytes.toBytes("testQualifierOne-0"), Bytes.toBytes("testQualifierOne-1"), 101 Bytes.toBytes("testQualifierOne-2"), Bytes.toBytes("testQualifierOne-3") }; 102 103 private static final byte[][] QUALIFIERS_TWO = 104 { Bytes.toBytes("testQualifierTwo-0"), Bytes.toBytes("testQualifierTwo-1"), 105 Bytes.toBytes("testQualifierTwo-2"), Bytes.toBytes("testQualifierTwo-3") }; 106 107 private static final byte[][] VALUES = 108 { Bytes.toBytes("testValueOne"), Bytes.toBytes("testValueTwo") }; 109 110 private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); 111 private static final HBaseRESTTestingUtility REST_TEST_UTIL = new HBaseRESTTestingUtility(); 112 private static Client client; 113 private static JAXBContext context; 114 private static Marshaller marshaller; 115 private static Unmarshaller unmarshaller; 116 private static long numRows = (long) ROWS_ONE.length + ROWS_TWO.length; 117 private static long colsPerRow = (long) FAMILIES.length * QUALIFIERS_ONE.length; 118 119 @BeforeAll 120 public static void setUpBeforeClass() throws Exception { 121 TEST_UTIL.startMiniCluster(3); 122 REST_TEST_UTIL.startServletContainer(TEST_UTIL.getConfiguration()); 123 context = JAXBContext.newInstance(CellModel.class, CellSetModel.class, RowModel.class, 124 ScannerModel.class); 125 marshaller = context.createMarshaller(); 126 unmarshaller = context.createUnmarshaller(); 127 client = new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort())); 128 Admin admin = TEST_UTIL.getAdmin(); 129 if (!admin.tableExists(TABLE)) { 130 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TABLE) 131 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILIES[0])) 132 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILIES[1])).build(); 133 admin.createTable(tableDescriptor); 134 Table table = TEST_UTIL.getConnection().getTable(TABLE); 135 // Insert first half 136 for (byte[] ROW : ROWS_ONE) { 137 Put p = new Put(ROW); 138 p.setDurability(Durability.SKIP_WAL); 139 for (byte[] QUALIFIER : QUALIFIERS_ONE) { 140 p.addColumn(FAMILIES[0], QUALIFIER, VALUES[0]); 141 } 142 table.put(p); 143 } 144 for (byte[] ROW : ROWS_TWO) { 145 Put p = new Put(ROW); 146 p.setDurability(Durability.SKIP_WAL); 147 for (byte[] QUALIFIER : QUALIFIERS_TWO) { 148 p.addColumn(FAMILIES[1], QUALIFIER, VALUES[1]); 149 } 150 table.put(p); 151 } 152 153 // Insert second half (reverse families) 154 for (byte[] ROW : ROWS_ONE) { 155 Put p = new Put(ROW); 156 p.setDurability(Durability.SKIP_WAL); 157 for (byte[] QUALIFIER : QUALIFIERS_ONE) { 158 p.addColumn(FAMILIES[1], QUALIFIER, VALUES[0]); 159 } 160 table.put(p); 161 } 162 for (byte[] ROW : ROWS_TWO) { 163 Put p = new Put(ROW); 164 p.setDurability(Durability.SKIP_WAL); 165 for (byte[] QUALIFIER : QUALIFIERS_TWO) { 166 p.addColumn(FAMILIES[0], QUALIFIER, VALUES[1]); 167 } 168 table.put(p); 169 } 170 171 // Delete the second qualifier from all rows and families 172 for (byte[] ROW : ROWS_ONE) { 173 Delete d = new Delete(ROW); 174 d.addColumns(FAMILIES[0], QUALIFIERS_ONE[1]); 175 d.addColumns(FAMILIES[1], QUALIFIERS_ONE[1]); 176 table.delete(d); 177 } 178 for (byte[] ROW : ROWS_TWO) { 179 Delete d = new Delete(ROW); 180 d.addColumns(FAMILIES[0], QUALIFIERS_TWO[1]); 181 d.addColumns(FAMILIES[1], QUALIFIERS_TWO[1]); 182 table.delete(d); 183 } 184 colsPerRow -= 2; 185 186 // Delete the second rows from both groups, one column at a time 187 for (byte[] QUALIFIER : QUALIFIERS_ONE) { 188 Delete d = new Delete(ROWS_ONE[1]); 189 d.addColumns(FAMILIES[0], QUALIFIER); 190 d.addColumns(FAMILIES[1], QUALIFIER); 191 table.delete(d); 192 } 193 for (byte[] QUALIFIER : QUALIFIERS_TWO) { 194 Delete d = new Delete(ROWS_TWO[1]); 195 d.addColumns(FAMILIES[0], QUALIFIER); 196 d.addColumns(FAMILIES[1], QUALIFIER); 197 table.delete(d); 198 } 199 numRows -= 2; 200 table.close(); 201 } 202 } 203 204 @AfterAll 205 public static void tearDownAfterClass() throws Exception { 206 REST_TEST_UTIL.shutdownServletContainer(); 207 TEST_UTIL.shutdownMiniCluster(); 208 } 209 210 private static void verifyScan(Scan s, long expectedRows, long expectedKeys) throws Exception { 211 ScannerModel model = ScannerModel.fromScan(s); 212 model.setBatch(Integer.MAX_VALUE); // fetch it all at once 213 StringWriter writer = new StringWriter(); 214 marshaller.marshal(model, writer); 215 LOG.debug(writer.toString()); 216 byte[] body = Bytes.toBytes(writer.toString()); 217 Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body); 218 assertEquals(201, response.getCode()); 219 String scannerURI = response.getLocation(); 220 assertNotNull(scannerURI); 221 222 // get a cell set 223 response = client.get(scannerURI, Constants.MIMETYPE_XML); 224 assertEquals(200, response.getCode()); 225 assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); 226 CellSetModel cells = 227 (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody())); 228 229 int rows = cells.getRows().size(); 230 assertEquals(expectedRows, rows, 231 "Unexpected number of rows! Expected " + expectedRows + " total but scanned " + rows); 232 for (RowModel row : cells.getRows()) { 233 int count = row.getCells().size(); 234 assertEquals(expectedKeys, count, 235 "Expected " + expectedKeys + " keys per row but " + "returned " + count); 236 } 237 238 // delete the scanner 239 response = client.delete(scannerURI); 240 assertEquals(200, response.getCode()); 241 } 242 243 private static void verifyScanFull(Scan s, KeyValue[] kvs) throws Exception { 244 ScannerModel model = ScannerModel.fromScan(s); 245 model.setBatch(Integer.MAX_VALUE); // fetch it all at once 246 StringWriter writer = new StringWriter(); 247 marshaller.marshal(model, writer); 248 LOG.debug(writer.toString()); 249 byte[] body = Bytes.toBytes(writer.toString()); 250 Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body); 251 assertEquals(201, response.getCode()); 252 String scannerURI = response.getLocation(); 253 assertNotNull(scannerURI); 254 255 // get a cell set 256 response = client.get(scannerURI, Constants.MIMETYPE_XML); 257 assertEquals(200, response.getCode()); 258 assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); 259 CellSetModel cellSet = 260 (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody())); 261 262 // delete the scanner 263 response = client.delete(scannerURI); 264 assertEquals(200, response.getCode()); 265 266 int row = 0; 267 int idx = 0; 268 Iterator<RowModel> i = cellSet.getRows().iterator(); 269 for (boolean done = true; done; row++) { 270 done = i.hasNext(); 271 if (!done) { 272 break; 273 } 274 275 RowModel rowModel = i.next(); 276 List<CellModel> cells = rowModel.getCells(); 277 if (cells.isEmpty()) { 278 break; 279 } 280 281 assertTrue(kvs.length >= idx + cells.size(), "Scanned too many keys! Only expected " 282 + kvs.length + " total but already scanned " + (cells.size() + idx)); 283 for (CellModel cell : cells) { 284 assertTrue(Bytes.equals(rowModel.getKey(), CellUtil.cloneRow(kvs[idx])), "Row mismatch"); 285 byte[][] split = CellUtil.parseColumn(cell.getColumn()); 286 assertTrue(Bytes.equals(split[0], CellUtil.cloneFamily(kvs[idx])), "Family mismatch"); 287 assertTrue(Bytes.equals(split[1], CellUtil.cloneQualifier(kvs[idx])), "Qualifier mismatch"); 288 assertTrue(Bytes.equals(cell.getValue(), CellUtil.cloneValue(kvs[idx])), "Value mismatch"); 289 idx++; 290 } 291 } 292 assertEquals(kvs.length, idx, "Expected " + kvs.length + " total keys but scanned " + idx); 293 } 294 295 private static void verifyScanNoEarlyOut(Scan s, long expectedRows, long expectedKeys) 296 throws Exception { 297 ScannerModel model = ScannerModel.fromScan(s); 298 model.setBatch(Integer.MAX_VALUE); // fetch it all at once 299 StringWriter writer = new StringWriter(); 300 marshaller.marshal(model, writer); 301 LOG.debug(writer.toString()); 302 byte[] body = Bytes.toBytes(writer.toString()); 303 Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body); 304 assertEquals(201, response.getCode()); 305 String scannerURI = response.getLocation(); 306 assertNotNull(scannerURI); 307 308 // get a cell set 309 response = client.get(scannerURI, Constants.MIMETYPE_XML); 310 assertEquals(200, response.getCode()); 311 assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); 312 CellSetModel cellSet = 313 (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody())); 314 315 // delete the scanner 316 response = client.delete(scannerURI); 317 assertEquals(200, response.getCode()); 318 319 Iterator<RowModel> i = cellSet.getRows().iterator(); 320 int j = 0; 321 for (boolean done = true; done; j++) { 322 done = i.hasNext(); 323 if (!done) { 324 break; 325 } 326 327 RowModel rowModel = i.next(); 328 List<CellModel> cells = rowModel.getCells(); 329 if (cells.isEmpty()) { 330 break; 331 } 332 333 assertTrue(expectedRows > j, "Scanned too many rows! Only expected " + expectedRows 334 + " total but already scanned " + (j + 1)); 335 assertEquals(expectedKeys, cells.size(), 336 "Expected " + expectedKeys + " keys per row but " + "returned " + cells.size()); 337 } 338 assertEquals(expectedRows, j, "Expected " + expectedRows + " rows but scanned " + j + " rows"); 339 } 340 341 @Test 342 public void testNoFilter() throws Exception { 343 // No filter 344 long expectedRows = numRows; 345 long expectedKeys = colsPerRow; 346 347 // Both families 348 Scan s = new Scan(); 349 verifyScan(s, expectedRows, expectedKeys); 350 351 // One family 352 s = new Scan(); 353 s.addFamily(FAMILIES[0]); 354 verifyScan(s, expectedRows, expectedKeys / 2); 355 } 356 357 @Test 358 public void testPrefixFilter() throws Exception { 359 // Grab rows from group one (half of total) 360 long expectedRows = numRows / 2; 361 long expectedKeys = colsPerRow; 362 Scan s = new Scan(); 363 s.setFilter(new PrefixFilter(Bytes.toBytes("testRowOne"))); 364 verifyScan(s, expectedRows, expectedKeys); 365 } 366 367 @Test 368 public void testPageFilter() throws Exception { 369 // KVs in first 6 rows 370 KeyValue[] expectedKVs = { 371 // testRowOne-0 372 new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 373 new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), 374 new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 375 new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 376 new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), 377 new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 378 // testRowOne-2 379 new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 380 new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), 381 new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 382 new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 383 new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), 384 new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 385 // testRowOne-3 386 new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 387 new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), 388 new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 389 new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 390 new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), 391 new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 392 // testRowTwo-0 393 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 394 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 395 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 396 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 397 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 398 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 399 // testRowTwo-2 400 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 401 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 402 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 403 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 404 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 405 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 406 // testRowTwo-3 407 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 408 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 409 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 410 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 411 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 412 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]) }; 413 414 // Grab all 6 rows 415 long expectedRows = 6; 416 long expectedKeys = colsPerRow; 417 Scan s = new Scan(); 418 s.setFilter(new PageFilter(expectedRows)); 419 verifyScan(s, expectedRows, expectedKeys); 420 s.setFilter(new PageFilter(expectedRows)); 421 verifyScanFull(s, expectedKVs); 422 423 // Grab first 4 rows (6 cols per row) 424 expectedRows = 4; 425 expectedKeys = colsPerRow; 426 s = new Scan(); 427 s.setFilter(new PageFilter(expectedRows)); 428 verifyScan(s, expectedRows, expectedKeys); 429 s.setFilter(new PageFilter(expectedRows)); 430 verifyScanFull(s, Arrays.copyOf(expectedKVs, 24)); 431 432 // Grab first 2 rows 433 expectedRows = 2; 434 expectedKeys = colsPerRow; 435 s = new Scan(); 436 s.setFilter(new PageFilter(expectedRows)); 437 verifyScan(s, expectedRows, expectedKeys); 438 s.setFilter(new PageFilter(expectedRows)); 439 verifyScanFull(s, Arrays.copyOf(expectedKVs, 12)); 440 441 // Grab first row 442 expectedRows = 1; 443 expectedKeys = colsPerRow; 444 s = new Scan(); 445 s.setFilter(new PageFilter(expectedRows)); 446 verifyScan(s, expectedRows, expectedKeys); 447 s.setFilter(new PageFilter(expectedRows)); 448 verifyScanFull(s, Arrays.copyOf(expectedKVs, 6)); 449 } 450 451 @Test 452 public void testInclusiveStopFilter() throws Exception { 453 // Grab rows from group one 454 455 // If we just use start/stop row, we get total/2 - 1 rows 456 long expectedRows = (numRows / 2) - 1; 457 long expectedKeys = colsPerRow; 458 Scan s = new Scan().withStartRow(Bytes.toBytes("testRowOne-0")) 459 .withStopRow(Bytes.toBytes("testRowOne-3")); 460 verifyScan(s, expectedRows, expectedKeys); 461 462 // Now use start row with inclusive stop filter 463 expectedRows = numRows / 2; 464 s = new Scan().withStartRow(Bytes.toBytes("testRowOne-0")); 465 s.setFilter(new InclusiveStopFilter(Bytes.toBytes("testRowOne-3"))); 466 verifyScan(s, expectedRows, expectedKeys); 467 468 // Grab rows from group two 469 470 // If we just use start/stop row, we get total/2 - 1 rows 471 expectedRows = (numRows / 2) - 1; 472 expectedKeys = colsPerRow; 473 s = new Scan().withStartRow(Bytes.toBytes("testRowTwo-0")) 474 .withStopRow(Bytes.toBytes("testRowTwo-3")); 475 verifyScan(s, expectedRows, expectedKeys); 476 477 // Now use start row with inclusive stop filter 478 expectedRows = numRows / 2; 479 s = new Scan().withStartRow(Bytes.toBytes("testRowTwo-0")); 480 s.setFilter(new InclusiveStopFilter(Bytes.toBytes("testRowTwo-3"))); 481 verifyScan(s, expectedRows, expectedKeys); 482 } 483 484 @Test 485 public void testQualifierFilter() throws Exception { 486 // Match two keys (one from each family) in half the rows 487 long expectedRows = numRows / 2; 488 long expectedKeys = 2; 489 Filter f = new QualifierFilter(CompareOperator.EQUAL, 490 new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); 491 Scan s = new Scan(); 492 s.setFilter(f); 493 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 494 495 // Match keys less than same qualifier 496 // Expect only two keys (one from each family) in half the rows 497 expectedRows = numRows / 2; 498 expectedKeys = 2; 499 f = new QualifierFilter(CompareOperator.LESS, 500 new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); 501 s = new Scan(); 502 s.setFilter(f); 503 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 504 505 // Match keys less than or equal. Expect four keys (two from each family) in half the rows 506 expectedRows = numRows / 2; 507 expectedKeys = 4; 508 f = new QualifierFilter(CompareOperator.LESS_OR_EQUAL, 509 new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); 510 s = new Scan(); 511 s.setFilter(f); 512 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 513 514 // Match keys not equal. Expect four keys (two from each family) 515 // Only look in first group of rows 516 expectedRows = numRows / 2; 517 expectedKeys = 4; 518 f = new QualifierFilter(CompareOperator.NOT_EQUAL, 519 new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); 520 s = 521 new Scan().withStartRow(HConstants.EMPTY_START_ROW).withStopRow(Bytes.toBytes("testRowTwo")); 522 s.setFilter(f); 523 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 524 525 // Match keys greater or equal. Expect four keys (two from each family) 526 // Only look in first group of rows 527 expectedRows = numRows / 2; 528 expectedKeys = 4; 529 f = new QualifierFilter(CompareOperator.GREATER_OR_EQUAL, 530 new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); 531 s = 532 new Scan().withStartRow(HConstants.EMPTY_START_ROW).withStopRow(Bytes.toBytes("testRowTwo")); 533 s.setFilter(f); 534 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 535 536 // Match keys greater. Expect two keys (one from each family) 537 // Only look in first group of rows 538 expectedRows = numRows / 2; 539 expectedKeys = 2; 540 f = new QualifierFilter(CompareOperator.GREATER, 541 new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); 542 s = 543 new Scan().withStartRow(HConstants.EMPTY_START_ROW).withStopRow(Bytes.toBytes("testRowTwo")); 544 s.setFilter(f); 545 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 546 547 // Match keys not equal to. Look across rows and fully validate the keys and ordering 548 // Expect varied numbers of keys, 4 per row in group one, 6 per row in group two 549 f = new QualifierFilter(CompareOperator.NOT_EQUAL, new BinaryComparator(QUALIFIERS_ONE[2])); 550 s = new Scan(); 551 s.setFilter(f); 552 553 KeyValue[] kvs = { 554 // testRowOne-0 555 new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 556 new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 557 new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 558 new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 559 // testRowOne-2 560 new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 561 new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 562 new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 563 new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 564 // testRowOne-3 565 new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 566 new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 567 new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 568 new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 569 // testRowTwo-0 570 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 571 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 572 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 573 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 574 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 575 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 576 // testRowTwo-2 577 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 578 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 579 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 580 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 581 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 582 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 583 // testRowTwo-3 584 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 585 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 586 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 587 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 588 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 589 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), }; 590 verifyScanFull(s, kvs); 591 592 // Test across rows and groups with a regex. Filter out "test*-2" 593 // Expect 4 keys per row across both groups 594 f = new QualifierFilter(CompareOperator.NOT_EQUAL, new RegexStringComparator("test.+-2")); 595 s = new Scan(); 596 s.setFilter(f); 597 598 kvs = new KeyValue[] { 599 // testRowOne-0 600 new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 601 new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 602 new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 603 new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 604 // testRowOne-2 605 new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 606 new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 607 new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 608 new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 609 // testRowOne-3 610 new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 611 new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 612 new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 613 new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 614 // testRowTwo-0 615 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 616 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 617 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 618 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 619 // testRowTwo-2 620 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 621 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 622 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 623 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 624 // testRowTwo-3 625 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 626 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 627 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 628 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), }; 629 verifyScanFull(s, kvs); 630 } 631 632 @Test 633 public void testRowFilter() throws Exception { 634 // Match a single row, all keys 635 long expectedRows = 1; 636 long expectedKeys = colsPerRow; 637 Filter f = 638 new RowFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("testRowOne-2"))); 639 Scan s = new Scan(); 640 s.setFilter(f); 641 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 642 643 // Match a two rows, one from each group, using regex 644 expectedRows = 2; 645 expectedKeys = colsPerRow; 646 f = new RowFilter(CompareOperator.EQUAL, new RegexStringComparator("testRow.+-2")); 647 s = new Scan(); 648 s.setFilter(f); 649 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 650 651 // Match rows less than 652 // Expect all keys in one row 653 expectedRows = 1; 654 expectedKeys = colsPerRow; 655 f = new RowFilter(CompareOperator.LESS, new BinaryComparator(Bytes.toBytes("testRowOne-2"))); 656 s = new Scan(); 657 s.setFilter(f); 658 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 659 660 // Match rows less than or equal 661 // Expect all keys in two rows 662 expectedRows = 2; 663 expectedKeys = colsPerRow; 664 f = new RowFilter(CompareOperator.LESS_OR_EQUAL, 665 new BinaryComparator(Bytes.toBytes("testRowOne-2"))); 666 s = new Scan(); 667 s.setFilter(f); 668 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 669 670 // Match rows not equal 671 // Expect all keys in all but one row 672 expectedRows = numRows - 1; 673 expectedKeys = colsPerRow; 674 f = 675 new RowFilter(CompareOperator.NOT_EQUAL, new BinaryComparator(Bytes.toBytes("testRowOne-2"))); 676 s = new Scan(); 677 s.setFilter(f); 678 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 679 680 // Match keys greater or equal 681 // Expect all keys in all but one row 682 expectedRows = numRows - 1; 683 expectedKeys = colsPerRow; 684 f = new RowFilter(CompareOperator.GREATER_OR_EQUAL, 685 new BinaryComparator(Bytes.toBytes("testRowOne-2"))); 686 s = new Scan(); 687 s.setFilter(f); 688 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 689 690 // Match keys greater 691 // Expect all keys in all but two rows 692 expectedRows = numRows - 2; 693 expectedKeys = colsPerRow; 694 f = new RowFilter(CompareOperator.GREATER, new BinaryComparator(Bytes.toBytes("testRowOne-2"))); 695 s = new Scan(); 696 s.setFilter(f); 697 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 698 699 // Match rows not equal to testRowTwo-2 700 // Look across rows and fully validate the keys and ordering 701 // Should see all keys in all rows but testRowTwo-2 702 f = 703 new RowFilter(CompareOperator.NOT_EQUAL, new BinaryComparator(Bytes.toBytes("testRowOne-2"))); 704 s = new Scan(); 705 s.setFilter(f); 706 707 KeyValue[] kvs = { 708 // testRowOne-0 709 new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 710 new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), 711 new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 712 new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 713 new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), 714 new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 715 // testRowOne-3 716 new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 717 new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), 718 new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 719 new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 720 new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), 721 new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 722 // testRowTwo-0 723 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 724 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 725 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 726 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 727 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 728 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 729 // testRowTwo-2 730 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 731 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 732 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 733 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 734 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 735 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 736 // testRowTwo-3 737 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 738 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 739 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 740 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 741 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 742 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), }; 743 verifyScanFull(s, kvs); 744 745 // Test across rows and groups with a regex 746 // Filter out everything that doesn't match "*-2" 747 // Expect all keys in two rows 748 f = new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2")); 749 s = new Scan(); 750 s.setFilter(f); 751 752 kvs = new KeyValue[] { 753 // testRowOne-2 754 new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 755 new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), 756 new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 757 new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 758 new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), 759 new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 760 // testRowTwo-2 761 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 762 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 763 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 764 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 765 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 766 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]) }; 767 verifyScanFull(s, kvs); 768 } 769 770 @Test 771 public void testValueFilter() throws Exception { 772 // Match group one rows 773 long expectedRows = numRows / 2; 774 long expectedKeys = colsPerRow; 775 Filter f = 776 new ValueFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("testValueOne"))); 777 Scan s = new Scan(); 778 s.setFilter(f); 779 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 780 781 // Match group two rows 782 expectedRows = numRows / 2; 783 expectedKeys = colsPerRow; 784 f = new ValueFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("testValueTwo"))); 785 s = new Scan(); 786 s.setFilter(f); 787 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 788 789 // Match all values using regex 790 expectedRows = numRows; 791 expectedKeys = colsPerRow; 792 f = new ValueFilter(CompareOperator.EQUAL, new RegexStringComparator("testValue((One)|(Two))")); 793 s = new Scan(); 794 s.setFilter(f); 795 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 796 797 // Match values less than 798 // Expect group one rows 799 expectedRows = numRows / 2; 800 expectedKeys = colsPerRow; 801 f = new ValueFilter(CompareOperator.LESS, new BinaryComparator(Bytes.toBytes("testValueTwo"))); 802 s = new Scan(); 803 s.setFilter(f); 804 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 805 806 // Match values less than or equal 807 // Expect all rows 808 expectedRows = numRows; 809 expectedKeys = colsPerRow; 810 f = new ValueFilter(CompareOperator.LESS_OR_EQUAL, 811 new BinaryComparator(Bytes.toBytes("testValueTwo"))); 812 s = new Scan(); 813 s.setFilter(f); 814 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 815 816 // Match values less than or equal 817 // Expect group one rows 818 expectedRows = numRows / 2; 819 expectedKeys = colsPerRow; 820 f = new ValueFilter(CompareOperator.LESS_OR_EQUAL, 821 new BinaryComparator(Bytes.toBytes("testValueOne"))); 822 s = new Scan(); 823 s.setFilter(f); 824 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 825 826 // Match values not equal 827 // Expect half the rows 828 expectedRows = numRows / 2; 829 expectedKeys = colsPerRow; 830 f = new ValueFilter(CompareOperator.NOT_EQUAL, 831 new BinaryComparator(Bytes.toBytes("testValueOne"))); 832 s = new Scan(); 833 s.setFilter(f); 834 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 835 836 // Match values greater or equal 837 // Expect all rows 838 expectedRows = numRows; 839 expectedKeys = colsPerRow; 840 f = new ValueFilter(CompareOperator.GREATER_OR_EQUAL, 841 new BinaryComparator(Bytes.toBytes("testValueOne"))); 842 s = new Scan(); 843 s.setFilter(f); 844 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 845 846 // Match values greater 847 // Expect half rows 848 expectedRows = numRows / 2; 849 expectedKeys = colsPerRow; 850 f = 851 new ValueFilter(CompareOperator.GREATER, new BinaryComparator(Bytes.toBytes("testValueOne"))); 852 s = new Scan(); 853 s.setFilter(f); 854 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 855 856 // Match values not equal to testValueOne 857 // Look across rows and fully validate the keys and ordering 858 // Should see all keys in all group two rows 859 f = new ValueFilter(CompareOperator.NOT_EQUAL, 860 new BinaryComparator(Bytes.toBytes("testValueOne"))); 861 s = new Scan(); 862 s.setFilter(f); 863 864 KeyValue[] kvs = { 865 // testRowTwo-0 866 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 867 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 868 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 869 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 870 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 871 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 872 // testRowTwo-2 873 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 874 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 875 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 876 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 877 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 878 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 879 // testRowTwo-3 880 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 881 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 882 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 883 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 884 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 885 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), }; 886 verifyScanFull(s, kvs); 887 } 888 889 @Test 890 public void testSkipFilter() throws Exception { 891 // Test for qualifier regex: "testQualifierOne-2" 892 // Should only get rows from second group, and all keys 893 Filter f = new SkipFilter(new QualifierFilter(CompareOperator.NOT_EQUAL, 894 new BinaryComparator(Bytes.toBytes("testQualifierOne-2")))); 895 Scan s = new Scan(); 896 s.setFilter(f); 897 898 KeyValue[] kvs = { 899 // testRowTwo-0 900 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 901 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 902 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 903 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 904 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 905 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 906 // testRowTwo-2 907 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 908 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 909 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 910 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 911 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 912 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 913 // testRowTwo-3 914 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 915 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 916 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 917 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 918 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 919 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), }; 920 verifyScanFull(s, kvs); 921 } 922 923 @Test 924 public void testFilterList() throws Exception { 925 // Test getting a single row, single key using Row, Qualifier, and Value 926 // regular expression and substring filters 927 // Use must pass all 928 List<Filter> filters = new ArrayList<>(3); 929 filters.add(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2"))); 930 filters.add(new QualifierFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2"))); 931 filters.add(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("One"))); 932 Filter f = new FilterList(Operator.MUST_PASS_ALL, filters); 933 Scan s = new Scan(); 934 s.addFamily(FAMILIES[0]); 935 s.setFilter(f); 936 KeyValue[] kvs = { new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]) }; 937 verifyScanFull(s, kvs); 938 939 // Test getting everything with a MUST_PASS_ONE filter including row, qf, 940 // val, regular expression and substring filters 941 filters.clear(); 942 filters.add(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(".+Two.+"))); 943 filters.add(new QualifierFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2"))); 944 filters.add(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("One"))); 945 f = new FilterList(Operator.MUST_PASS_ONE, filters); 946 s = new Scan(); 947 s.setFilter(f); 948 verifyScanNoEarlyOut(s, numRows, colsPerRow); 949 } 950 951 @Test 952 public void testFirstKeyOnlyFilter() throws Exception { 953 Scan s = new Scan(); 954 s.setFilter(new FirstKeyOnlyFilter()); 955 // Expected KVs, the first KV from each of the remaining 6 rows 956 KeyValue[] kvs = { new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 957 new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 958 new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 959 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 960 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 961 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]) }; 962 verifyScanFull(s, kvs); 963 } 964 965 @Test 966 public void testMultiRowRangeFilter() throws Exception { 967 long expectedRows = 2; 968 long expectedKeys = colsPerRow; 969 List<MultiRowRangeFilter.RowRange> ranges = new ArrayList<>(); 970 // Both return only the third element, as the second one is deleted during initialization. 971 ranges.add(new MultiRowRangeFilter.RowRange(ROWS_ONE[1], true, ROWS_ONE[2], true)); 972 ranges.add(new MultiRowRangeFilter.RowRange(ROWS_TWO[0], false, ROWS_TWO[3], false)); 973 974 Scan s = new Scan(); 975 s.setFilter(new MultiRowRangeFilter(ranges)); 976 verifyScan(s, expectedRows, expectedKeys); 977 } 978 979 @Test 980 public void testFuzzyRowFilter() throws Exception { 981 long expectedRows = 4; 982 long expectedKeys = colsPerRow; 983 List<Pair<byte[], byte[]>> fuzzyKeys = new ArrayList<>(); 984 985 // Exact match for ROWS_ONE[0] (one row) 986 byte[] rowOneMask = new byte[ROWS_ONE[0].length]; 987 Arrays.fill(rowOneMask, (byte) 0); 988 fuzzyKeys.add(new Pair<>(ROWS_ONE[0], rowOneMask)); 989 // All ROW_TWO keys (three rows) 990 byte[] rowTwoMask = new byte[ROWS_TWO[0].length]; 991 Arrays.fill(rowTwoMask, (byte) 0); 992 rowTwoMask[rowTwoMask.length - 1] = (byte) 1; 993 fuzzyKeys.add(new Pair<>(ROWS_TWO[2], rowTwoMask)); 994 995 Scan s = new Scan(); 996 s.setFilter(new FuzzyRowFilter(fuzzyKeys)); 997 verifyScan(s, expectedRows, expectedKeys); 998 } 999}