001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.junit.jupiter.api.Assertions.assertEquals; 021import static org.junit.jupiter.api.Assertions.assertFalse; 022import static org.junit.jupiter.api.Assertions.assertTrue; 023import static org.junit.jupiter.api.Assertions.fail; 024 025import java.io.IOException; 026import java.lang.management.ManagementFactory; 027import java.util.Iterator; 028import java.util.List; 029import org.apache.hadoop.conf.Configuration; 030import org.apache.hadoop.fs.FileSystem; 031import org.apache.hadoop.hbase.ChoreService; 032import org.apache.hadoop.hbase.CoordinatedStateManager; 033import org.apache.hadoop.hbase.HBaseConfiguration; 034import org.apache.hadoop.hbase.HBaseTestingUtil; 035import org.apache.hadoop.hbase.HConstants; 036import org.apache.hadoop.hbase.Server; 037import org.apache.hadoop.hbase.ServerName; 038import org.apache.hadoop.hbase.Waiter; 039import org.apache.hadoop.hbase.client.AsyncClusterConnection; 040import org.apache.hadoop.hbase.client.Connection; 041import org.apache.hadoop.hbase.io.hfile.BlockCache; 042import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; 043import org.apache.hadoop.hbase.io.hfile.CacheStats; 044import org.apache.hadoop.hbase.io.hfile.Cacheable; 045import org.apache.hadoop.hbase.io.hfile.CachedBlock; 046import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache; 047import org.apache.hadoop.hbase.io.util.MemorySizeUtil; 048import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext; 049import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult; 050import org.apache.hadoop.hbase.testclassification.MediumTests; 051import org.apache.hadoop.hbase.testclassification.RegionServerTests; 052import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 053import org.junit.jupiter.api.Tag; 054import org.junit.jupiter.api.Test; 055 056@Tag(RegionServerTests.TAG) 057@Tag(MediumTests.TAG) 058public class TestHeapMemoryManager { 059 060 private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); 061 062 private long maxHeapSize = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax(); 063 064 @Test 065 public void testAutoTunerShouldBeOffWhenMaxMinRangesForMemstoreIsNotGiven() throws Exception { 066 Configuration conf = HBaseConfiguration.create(); 067 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.02f); 068 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f); 069 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.03f); 070 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 071 HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0), 072 new MemstoreFlusherStub(0), new RegionServerStub(conf), regionServerAccounting); 073 assertFalse(manager.isTunerOn()); 074 } 075 076 @Test 077 public void testAutoTunerShouldBeOffWhenMaxMinRangesForBlockCacheIsNotGiven() throws Exception { 078 Configuration conf = HBaseConfiguration.create(); 079 conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.02f); 080 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 081 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.03f); 082 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 083 HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0), 084 new MemstoreFlusherStub(0), new RegionServerStub(conf), regionServerAccounting); 085 assertFalse(manager.isTunerOn()); 086 } 087 088 @Test 089 public void testWhenMemstoreAndBlockCacheMaxMinChecksFails() throws Exception { 090 BlockCacheStub blockCache = new BlockCacheStub(0); 091 Configuration conf = HBaseConfiguration.create(); 092 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 093 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.06f); 094 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 095 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub(0); 096 try { 097 new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf), 098 regionServerAccounting); 099 fail(); 100 } catch (RuntimeException e) { 101 } 102 conf = HBaseConfiguration.create(); 103 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.2f); 104 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 105 try { 106 new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf), 107 regionServerAccounting); 108 fail(); 109 } catch (RuntimeException e) { 110 } 111 } 112 113 @Test 114 public void testWhenClusterIsWriteHeavyWithEmptyMemstore() throws Exception { 115 Configuration conf = HBaseConfiguration.create(); 116 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 117 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 118 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 119 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 120 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 121 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 122 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 123 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 124 // Empty block cache and memstore 125 blockCache.setTestBlockSize(0); 126 regionServerAccounting.setTestMemstoreSize(0); 127 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 128 // Let the system start with default values for memstore heap and block cache size. 129 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 130 new RegionServerStub(conf), regionServerAccounting); 131 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 132 long oldBlockCacheSize = blockCache.maxSize; 133 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 134 heapMemoryManager.start(choreService); 135 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK; 136 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 137 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 138 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 139 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 140 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 141 // Allow the tuner to run once and do necessary memory up 142 Thread.sleep(1500); 143 // No changes should be made by tuner as we already have lot of empty space 144 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 145 assertEquals(oldBlockCacheSize, blockCache.maxSize); 146 } 147 148 @Test 149 public void testHeapMemoryManagerWhenOffheapFlushesHappenUnderReadHeavyCase() throws Exception { 150 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 151 Configuration conf = HBaseConfiguration.create(); 152 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f); 153 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 154 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 155 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 156 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 157 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 158 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 159 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true); 160 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 161 // Empty memstore and but nearly filled block cache 162 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 163 regionServerAccounting.setTestMemstoreSize(0); 164 // Let the system start with default values for memstore heap and block cache size. 165 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 166 new RegionServerStub(conf), regionServerAccounting); 167 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 168 long oldBlockCacheSize = blockCache.maxSize; 169 float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE; 170 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 171 heapMemoryManager.start(choreService); 172 blockCache.evictBlock(null); 173 blockCache.evictBlock(null); 174 blockCache.evictBlock(null); 175 // do some offheap flushes also. So there should be decrease in memstore but 176 // not as that when we don't have offheap flushes 177 memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK; 178 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 179 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 180 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 181 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 182 // Allow the tuner to run once and do necessary memory up 183 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 184 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 185 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 186 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 187 oldBlockCacheSize = blockCache.maxSize; 188 // Do some more evictions before the next run of HeapMemoryTuner 189 blockCache.evictBlock(null); 190 // Allow the tuner to run once and do necessary memory up 191 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 192 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 193 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 194 } 195 196 @Test 197 public void testHeapMemoryManagerWithOffheapMemstoreAndMixedWorkload() throws Exception { 198 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 199 Configuration conf = HBaseConfiguration.create(); 200 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f); 201 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 202 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 203 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 204 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 205 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 206 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 207 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true); 208 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 209 // Empty memstore and but nearly filled block cache 210 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 211 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 212 // Let the system start with default values for memstore heap and block cache size. 213 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 214 new RegionServerStub(conf), regionServerAccounting); 215 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 216 long oldBlockCacheSize = blockCache.maxSize; 217 float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE; 218 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 219 heapMemoryManager.start(choreService); 220 blockCache.evictBlock(null); 221 blockCache.evictBlock(null); 222 blockCache.evictBlock(null); 223 // do some offheap flushes also. So there should be decrease in memstore but 224 // not as that when we don't have offheap flushes 225 memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK; 226 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 227 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 228 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 229 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 230 // Allow the tuner to run once and do necessary memory up 231 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 232 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 233 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 234 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 235 oldBlockCacheSize = blockCache.maxSize; 236 // change memstore size 237 // regionServerAccounting.setTestMemstoreSize((long)(maxHeapSize * 0.4 * 0.8)); 238 // The memstore size would have decreased. Now again do some flushes and ensure the 239 // flushes are due to onheap overhead. This should once again call for increase in 240 // memstore size but that increase should be to the safe size 241 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK; 242 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 243 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 244 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 245 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 246 // Allow the tuner to run once and do necessary memory up 247 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 248 assertHeapSpaceDelta(maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 249 assertHeapSpaceDelta(-maxStepValue, oldBlockCacheSize, blockCache.maxSize); 250 } 251 252 @Test 253 public void testWhenClusterIsReadHeavyWithEmptyBlockCache() throws Exception { 254 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 255 Configuration conf = HBaseConfiguration.create(); 256 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 257 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 258 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 259 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 260 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 261 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 262 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 263 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 264 // Empty block cache and memstore 265 blockCache.setTestBlockSize(0); 266 regionServerAccounting.setTestMemstoreSize(0); 267 // Let the system start with default values for memstore heap and block cache size. 268 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 269 new RegionServerStub(conf), regionServerAccounting); 270 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 271 long oldBlockCacheSize = blockCache.maxSize; 272 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 273 heapMemoryManager.start(choreService); 274 blockCache.evictBlock(null); 275 blockCache.evictBlock(null); 276 blockCache.evictBlock(null); 277 // Allow the tuner to run once and do necessary memory up 278 Thread.sleep(1500); 279 // No changes should be made by tuner as we already have lot of empty space 280 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 281 assertEquals(oldBlockCacheSize, blockCache.maxSize); 282 } 283 284 @Test 285 public void testWhenClusterIsWriteHeavy() throws Exception { 286 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 287 Configuration conf = HBaseConfiguration.create(); 288 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 289 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 290 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 291 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 292 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 293 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 294 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 295 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 296 // Empty block cache and but nearly filled memstore 297 blockCache.setTestBlockSize(0); 298 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 299 // Let the system start with default values for memstore heap and block cache size. 300 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 301 new RegionServerStub(conf), regionServerAccounting); 302 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 303 long oldBlockCacheSize = blockCache.maxSize; 304 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 305 heapMemoryManager.start(choreService); 306 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 307 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 308 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 309 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 310 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 311 // Allow the tuner to run once and do necessary memory up 312 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 313 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 314 memStoreFlusher.memstoreSize); 315 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 316 blockCache.maxSize); 317 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 318 oldBlockCacheSize = blockCache.maxSize; 319 // Do some more flushes before the next run of HeapMemoryTuner 320 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 321 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 322 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 323 // Allow the tuner to run once and do necessary memory up 324 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 325 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 326 memStoreFlusher.memstoreSize); 327 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 328 blockCache.maxSize); 329 } 330 331 @Test 332 public void testWhenClusterIsWriteHeavyWithOffheapMemstore() throws Exception { 333 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 334 Configuration conf = HBaseConfiguration.create(); 335 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 336 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 337 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 338 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 339 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 340 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 341 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 342 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 343 // Empty block cache and but nearly filled memstore 344 blockCache.setTestBlockSize(0); 345 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 346 // Let the system start with default values for memstore heap and block cache size. 347 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 348 new RegionServerStub(conf), regionServerAccounting); 349 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 350 long oldBlockCacheSize = blockCache.maxSize; 351 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 352 heapMemoryManager.start(choreService); 353 // this should not change anything with onheap memstore 354 memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK; 355 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 356 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 357 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 358 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 359 // Allow the tuner to run once and do necessary memory up 360 Thread.sleep(1500); 361 // No changes should be made by tuner as we already have lot of empty space 362 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 363 assertEquals(oldBlockCacheSize, blockCache.maxSize); 364 } 365 366 @Test 367 public void testWhenClusterIsReadHeavy() throws Exception { 368 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 369 Configuration conf = HBaseConfiguration.create(); 370 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f); 371 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 372 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 373 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 374 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 375 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 376 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 377 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 378 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 379 // Empty memstore and but nearly filled block cache 380 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 381 regionServerAccounting.setTestMemstoreSize(0); 382 // Let the system start with default values for memstore heap and block cache size. 383 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 384 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 385 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 386 long oldBlockCacheSize = blockCache.maxSize; 387 long oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10; 388 long maxTuneSize = oldMemstoreHeapSize - (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2; 389 float maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize; 390 maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE 391 ? DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE 392 : maxStepValue; 393 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 394 heapMemoryManager.start(choreService); 395 blockCache.evictBlock(null); 396 blockCache.evictBlock(null); 397 blockCache.evictBlock(null); 398 // Allow the tuner to run once and do necessary memory up 399 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 400 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 401 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 402 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 403 oldBlockCacheSize = blockCache.maxSize; 404 oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10; 405 maxTuneSize = oldMemstoreHeapSize - (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2; 406 maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize; 407 maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE 408 ? DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE 409 : maxStepValue; 410 // Do some more evictions before the next run of HeapMemoryTuner 411 blockCache.evictBlock(null); 412 // Allow the tuner to run once and do necessary memory up 413 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 414 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 415 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 416 } 417 418 @Test 419 public void testWhenClusterIsHavingMoreWritesThanReads() throws Exception { 420 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 421 Configuration conf = HBaseConfiguration.create(); 422 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 423 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 424 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 425 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 426 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 427 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 428 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 429 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 430 // Both memstore and block cache are nearly filled 431 blockCache.setTestBlockSize(0); 432 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 433 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 434 // Let the system start with default values for memstore heap and block cache size. 435 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 436 new RegionServerStub(conf), regionServerAccounting); 437 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 438 long oldBlockCacheSize = blockCache.maxSize; 439 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 440 heapMemoryManager.start(choreService); 441 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 442 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 443 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 444 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 445 blockCache.evictBlock(null); 446 // Allow the tuner to run once and do necessary memory up 447 Thread.sleep(1500); 448 // No changes should happen as there is undefined increase in flushes and evictions 449 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 450 assertEquals(oldBlockCacheSize, blockCache.maxSize); 451 // Do some more flushes before the next run of HeapMemoryTuner 452 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 453 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 454 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 455 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 456 // Allow the tuner to run once and do necessary memory up 457 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 458 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 459 memStoreFlusher.memstoreSize); 460 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 461 blockCache.maxSize); 462 } 463 464 @Test 465 public void testBlockedFlushesIncreaseMemstoreInSteadyState() throws Exception { 466 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 467 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 468 Configuration conf = HBaseConfiguration.create(); 469 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 470 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 471 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 472 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 473 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 474 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 475 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 476 // Both memstore and block cache are nearly filled 477 blockCache.setTestBlockSize(0); 478 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 479 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 480 // Let the system start with default values for memstore heap and block cache size. 481 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 482 new RegionServerStub(conf), regionServerAccounting); 483 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 484 long oldBlockCacheSize = blockCache.maxSize; 485 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 486 heapMemoryManager.start(choreService); 487 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 488 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 489 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 490 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 491 blockCache.evictBlock(null); 492 blockCache.evictBlock(null); 493 // Allow the tuner to run once and do necessary memory up 494 Thread.sleep(1500); 495 // No changes should happen as there is undefined increase in flushes and evictions 496 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 497 assertEquals(oldBlockCacheSize, blockCache.maxSize); 498 // Flushes that block updates 499 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK; 500 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 501 blockCache.evictBlock(null); 502 blockCache.evictBlock(null); 503 blockCache.evictBlock(null); 504 blockCache.evictBlock(null); 505 // Allow the tuner to run once and do necessary memory up 506 Thread.sleep(1500); 507 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 508 memStoreFlusher.memstoreSize); 509 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 510 blockCache.maxSize); 511 } 512 513 @Test 514 public void testPluggingInHeapMemoryTuner() throws Exception { 515 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 516 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 517 Configuration conf = HBaseConfiguration.create(); 518 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.78f); 519 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.05f); 520 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f); 521 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.02f); 522 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 523 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 524 conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, 525 HeapMemoryTuner.class); 526 // Let the system start with default values for memstore heap and block cache size. 527 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 528 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 529 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 530 heapMemoryManager.start(choreService); 531 // Now we wants to be in write mode. Set bigger memstore size from CustomHeapMemoryTuner 532 CustomHeapMemoryTuner.memstoreSize = 0.78f; 533 CustomHeapMemoryTuner.blockCacheSize = 0.02f; 534 // Allow the tuner to run once and do necessary memory up 535 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 536 assertHeapSpace(0.78f, memStoreFlusher.memstoreSize);// Memstore 537 assertHeapSpace(0.02f, blockCache.maxSize);// BlockCache 538 // Now we wants to be in read mode. Set bigger memstore size from CustomHeapMemoryTuner 539 CustomHeapMemoryTuner.blockCacheSize = 0.75f; 540 CustomHeapMemoryTuner.memstoreSize = 0.05f; 541 // Allow the tuner to run once and do necessary memory up 542 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 543 assertHeapSpace(0.75f, blockCache.maxSize);// BlockCache 544 assertHeapSpace(0.05f, memStoreFlusher.memstoreSize);// Memstore 545 } 546 547 @Test 548 public void testWhenSizeGivenByHeapTunerGoesOutsideRange() throws Exception { 549 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 550 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 551 Configuration conf = HBaseConfiguration.create(); 552 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f); 553 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f); 554 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 555 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f); 556 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 557 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 558 conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, 559 HeapMemoryTuner.class); 560 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 561 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 562 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 563 heapMemoryManager.start(choreService); 564 CustomHeapMemoryTuner.memstoreSize = 0.78f; 565 CustomHeapMemoryTuner.blockCacheSize = 0.02f; 566 Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up 567 // Even if the tuner says to set the memstore to 78%, HBase makes it as 70% as that is the 568 // upper bound. Same with block cache as 10% is the lower bound. 569 assertHeapSpace(0.7f, memStoreFlusher.memstoreSize); 570 assertHeapSpace(0.1f, blockCache.maxSize); 571 } 572 573 @Test 574 public void testWhenCombinedHeapSizesFromTunerGoesOutSideMaxLimit() throws Exception { 575 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 576 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 577 Configuration conf = HBaseConfiguration.create(); 578 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f); 579 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f); 580 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 581 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f); 582 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 583 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 584 conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, 585 HeapMemoryTuner.class); 586 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 587 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 588 long oldMemstoreSize = memStoreFlusher.memstoreSize; 589 long oldBlockCacheSize = blockCache.maxSize; 590 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 591 heapMemoryManager.start(choreService); 592 CustomHeapMemoryTuner.memstoreSize = 0.7f; 593 CustomHeapMemoryTuner.blockCacheSize = 0.3f; 594 // Allow the tuner to run once and do necessary memory up 595 Thread.sleep(1500); 596 assertEquals(oldMemstoreSize, memStoreFlusher.memstoreSize); 597 assertEquals(oldBlockCacheSize, blockCache.maxSize); 598 } 599 600 private void assertHeapSpace(float expectedHeapPercentage, long currentHeapSpace) { 601 long expected = (long) (this.maxHeapSize * expectedHeapPercentage); 602 assertEquals(expected, currentHeapSpace); 603 } 604 605 private void assertHeapSpaceDelta(double expectedDeltaPercent, long oldHeapSpace, 606 long newHeapSpace) { 607 double expctedMinDelta = (double) (this.maxHeapSize * expectedDeltaPercent); 608 // Tolerable error 609 double error = 0.95; 610 if (expectedDeltaPercent > 0) { 611 assertTrue(expctedMinDelta * error <= (double) (newHeapSpace - oldHeapSpace)); 612 assertTrue(expctedMinDelta / error >= (double) (newHeapSpace - oldHeapSpace)); 613 } else { 614 assertTrue(-expctedMinDelta * error <= (double) (oldHeapSpace - newHeapSpace)); 615 assertTrue(-expctedMinDelta / error >= (double) (oldHeapSpace - newHeapSpace)); 616 } 617 } 618 619 private void waitForTune(final MemstoreFlusherStub memStoreFlusher, 620 final long oldMemstoreHeapSize) throws Exception { 621 // Allow the tuner to run once and do necessary memory up 622 UTIL.waitFor(10000, new Waiter.Predicate<Exception>() { 623 @Override 624 public boolean evaluate() throws Exception { 625 return oldMemstoreHeapSize != memStoreFlusher.memstoreSize; 626 } 627 }); 628 } 629 630 private static class BlockCacheStub implements ResizableBlockCache { 631 CacheStats stats = new CacheStats("test"); 632 long maxSize = 0; 633 private long testBlockSize = 0; 634 635 public BlockCacheStub(long size) { 636 this.maxSize = size; 637 } 638 639 @Override 640 public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) { 641 642 } 643 644 @Override 645 public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) { 646 647 } 648 649 @Override 650 public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat, 651 boolean updateCacheMetrics) { 652 return null; 653 } 654 655 @Override 656 public boolean evictBlock(BlockCacheKey cacheKey) { 657 stats.evicted(0, cacheKey != null ? cacheKey.isPrimary() : true); 658 return false; 659 } 660 661 @Override 662 public int evictBlocksByHfileName(String hfileName) { 663 stats.evicted(0, true); // Just assuming only one block for file here. 664 return 0; 665 } 666 667 @Override 668 public CacheStats getStats() { 669 return this.stats; 670 } 671 672 @Override 673 public void shutdown() { 674 675 } 676 677 @Override 678 public long size() { 679 return 0; 680 } 681 682 @Override 683 public long getMaxSize() { 684 return 0; 685 } 686 687 @Override 688 public long getFreeSize() { 689 return 0; 690 } 691 692 @Override 693 public long getCurrentSize() { 694 return this.testBlockSize; 695 } 696 697 @Override 698 public long getCurrentDataSize() { 699 return 0; 700 } 701 702 @Override 703 public long getBlockCount() { 704 return 0; 705 } 706 707 @Override 708 public long getDataBlockCount() { 709 return 0; 710 } 711 712 @Override 713 public void setMaxSize(long size) { 714 this.maxSize = size; 715 } 716 717 @Override 718 public Iterator<CachedBlock> iterator() { 719 return null; 720 } 721 722 @Override 723 public BlockCache[] getBlockCaches() { 724 return null; 725 } 726 727 public void setTestBlockSize(long testBlockSize) { 728 this.testBlockSize = testBlockSize; 729 } 730 } 731 732 private static class MemstoreFlusherStub implements FlushRequester { 733 734 long memstoreSize; 735 736 FlushRequestListener listener; 737 738 FlushType flushType = FlushType.NORMAL; 739 740 public MemstoreFlusherStub(long memstoreSize) { 741 this.memstoreSize = memstoreSize; 742 } 743 744 @Override 745 public boolean requestFlush(HRegion region, FlushLifeCycleTracker tracker) { 746 this.listener.flushRequested(flushType, region); 747 return true; 748 } 749 750 @Override 751 public boolean requestFlush(HRegion region, List<byte[]> families, 752 FlushLifeCycleTracker tracker) { 753 return true; 754 } 755 756 @Override 757 public boolean requestDelayedFlush(HRegion region, long delay) { 758 return true; 759 } 760 761 @Override 762 public void registerFlushRequestListener(FlushRequestListener listener) { 763 this.listener = listener; 764 } 765 766 @Override 767 public boolean unregisterFlushRequestListener(FlushRequestListener listener) { 768 return false; 769 } 770 771 @Override 772 public void setGlobalMemStoreLimit(long globalMemStoreSize) { 773 this.memstoreSize = globalMemStoreSize; 774 } 775 } 776 777 private static class RegionServerStub implements Server { 778 private Configuration conf; 779 private boolean stopped = false; 780 781 public RegionServerStub(Configuration conf) { 782 this.conf = conf; 783 } 784 785 @Override 786 public void abort(String why, Throwable e) { 787 788 } 789 790 @Override 791 public boolean isAborted() { 792 return false; 793 } 794 795 @Override 796 public void stop(String why) { 797 this.stopped = true; 798 } 799 800 @Override 801 public boolean isStopped() { 802 return this.stopped; 803 } 804 805 @Override 806 public Configuration getConfiguration() { 807 return this.conf; 808 } 809 810 @Override 811 public ZKWatcher getZooKeeper() { 812 return null; 813 } 814 815 @Override 816 public CoordinatedStateManager getCoordinatedStateManager() { 817 return null; 818 } 819 820 @Override 821 public Connection getConnection() { 822 return null; 823 } 824 825 @Override 826 public ServerName getServerName() { 827 return ServerName.valueOf("server1", 4000, 12345); 828 } 829 830 @Override 831 public ChoreService getChoreService() { 832 return null; 833 } 834 835 @Override 836 public FileSystem getFileSystem() { 837 return null; 838 } 839 840 @Override 841 public boolean isStopping() { 842 return false; 843 } 844 845 @Override 846 public Connection createConnection(Configuration conf) throws IOException { 847 return null; 848 } 849 850 @Override 851 public AsyncClusterConnection getAsyncClusterConnection() { 852 return null; 853 } 854 } 855 856 static class CustomHeapMemoryTuner implements HeapMemoryTuner { 857 static float blockCacheSize = 0.4f; 858 static float memstoreSize = 0.4f; 859 860 @Override 861 public Configuration getConf() { 862 return null; 863 } 864 865 @Override 866 public void setConf(Configuration arg0) { 867 868 } 869 870 @Override 871 public TunerResult tune(TunerContext context) { 872 TunerResult result = new TunerResult(true); 873 result.setBlockCacheSize(blockCacheSize); 874 result.setMemStoreSize(memstoreSize); 875 return result; 876 } 877 } 878 879 private static class RegionServerAccountingStub extends RegionServerAccounting { 880 boolean offheap; 881 882 public RegionServerAccountingStub(Configuration conf) { 883 super(conf); 884 } 885 886 public RegionServerAccountingStub(Configuration conf, boolean offheap) { 887 super(conf); 888 this.offheap = offheap; 889 } 890 891 private long testMemstoreSize = 0; 892 893 @Override 894 public long getGlobalMemStoreDataSize() { 895 return testMemstoreSize; 896 } 897 898 @Override 899 public long getGlobalMemStoreHeapSize() { 900 return testMemstoreSize; 901 } 902 903 @Override 904 public boolean isOffheap() { 905 return offheap; 906 } 907 908 public void setTestMemstoreSize(long testMemstoreSize) { 909 this.testMemstoreSize = testMemstoreSize; 910 } 911 } 912}