001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.crypto.tls;
019
020import static java.util.Objects.requireNonNull;
021
022import java.io.File;
023import java.io.IOException;
024import java.lang.invoke.MethodHandles;
025import java.nio.charset.StandardCharsets;
026import java.nio.file.Files;
027import java.nio.file.StandardCopyOption;
028import java.nio.file.StandardOpenOption;
029import java.security.GeneralSecurityException;
030import java.security.KeyPair;
031import java.security.Security;
032import java.security.cert.X509Certificate;
033import java.util.Arrays;
034import org.apache.hadoop.conf.Configuration;
035import org.apache.yetus.audience.InterfaceAudience;
036import org.bouncycastle.asn1.x500.X500Name;
037import org.bouncycastle.asn1.x500.X500NameBuilder;
038import org.bouncycastle.asn1.x500.style.BCStyle;
039import org.bouncycastle.asn1.x509.GeneralName;
040import org.bouncycastle.asn1.x509.GeneralNames;
041import org.bouncycastle.jce.provider.BouncyCastleProvider;
042import org.bouncycastle.operator.OperatorCreationException;
043
044/**
045 * This class simplifies the creation of certificates and private keys for SSL/TLS connections.
046 * <p/>
047 * This file has been copied from the Apache ZooKeeper project.
048 * @see <a href=
049 *      "https://github.com/apache/zookeeper/blob/c74658d398cdc1d207aa296cb6e20de00faec03e/zookeeper-server/src/test/java/org/apache/zookeeper/common/X509TestContext.java">Base
050 *      revision</a>
051 */
052@InterfaceAudience.Private
053public final class X509TestContext {
054
055  private static final String TRUST_STORE_PREFIX = "hbase_test_ca";
056  private static final String KEY_STORE_PREFIX = "hbase_test_key";
057
058  private final File tempDir;
059  private final Configuration conf;
060
061  private X509Certificate trustStoreCertificate;
062  private final char[] trustStorePassword;
063  private KeyPair trustStoreKeyPair;
064  private File trustStoreJksFile;
065  private File trustStorePemFile;
066  private File trustStorePkcs12File;
067  private File trustStoreBcfksFile;
068
069  private KeyPair keyStoreKeyPair;
070  private X509Certificate keyStoreCertificate;
071  private final char[] keyStorePassword;
072  private File keyStoreJksFile;
073  private File keyStorePemFile;
074  private File keyStorePkcs12File;
075  private File keyStoreBcfksFile;
076
077  /**
078   * Constructor is intentionally private, use the Builder class instead.
079   * @param conf               the configuration
080   * @param tempDir            the directory in which key store and trust store temp files will be
081   *                           written.
082   * @param trustStoreKeyPair  the key pair for the trust store.
083   * @param trustStorePassword the password to protect a JKS trust store (ignored for PEM trust
084   *                           stores).
085   * @param keyStoreKeyPair    the key pair for the key store.
086   * @param keyStorePassword   the password to protect the key store private key.
087   */
088  private X509TestContext(Configuration conf, File tempDir, KeyPair trustStoreKeyPair,
089    char[] trustStorePassword, KeyPair keyStoreKeyPair, char[] keyStorePassword)
090    throws IOException, GeneralSecurityException, OperatorCreationException {
091    if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) {
092      throw new IllegalStateException("BC Security provider was not found");
093    }
094    this.conf = conf;
095    this.tempDir = requireNonNull(tempDir);
096    if (!tempDir.isDirectory()) {
097      throw new IllegalArgumentException("Not a directory: " + tempDir);
098    }
099
100    this.trustStoreKeyPair = trustStoreKeyPair;
101    this.trustStorePassword = requireNonNull(trustStorePassword);
102    this.keyStoreKeyPair = requireNonNull(keyStoreKeyPair);
103    this.keyStorePassword = requireNonNull(keyStorePassword);
104
105    createCertificates();
106
107    trustStorePkcs12File = null;
108    trustStorePemFile = null;
109    trustStoreJksFile = null;
110    keyStorePkcs12File = null;
111    keyStorePemFile = null;
112    keyStoreJksFile = null;
113  }
114
115  /**
116   * Used by {@link #cloneWithNewKeystoreCert(X509Certificate)}. Should set all fields except
117   * generated keystore path fields
118   */
119  private X509TestContext(File tempDir, Configuration conf, X509Certificate trustStoreCertificate,
120    char[] trustStorePassword, KeyPair trustStoreKeyPair, File trustStoreJksFile,
121    File trustStorePemFile, File trustStorePkcs12File, KeyPair keyStoreKeyPair,
122    char[] keyStorePassword, X509Certificate keyStoreCertificate) {
123    this.tempDir = tempDir;
124    this.conf = conf;
125    this.trustStoreCertificate = trustStoreCertificate;
126    this.trustStorePassword = trustStorePassword;
127    this.trustStoreKeyPair = trustStoreKeyPair;
128    this.trustStoreJksFile = trustStoreJksFile;
129    this.trustStorePemFile = trustStorePemFile;
130    this.trustStorePkcs12File = trustStorePkcs12File;
131    this.keyStoreKeyPair = keyStoreKeyPair;
132    this.keyStoreCertificate = keyStoreCertificate;
133    this.keyStorePassword = keyStorePassword;
134    keyStorePkcs12File = null;
135    keyStorePemFile = null;
136    keyStoreJksFile = null;
137  }
138
139  /**
140   * Generates a new certificate using this context's CA and keystoreKeyPair. By default, the cert
141   * will have localhost in the subjectAltNames. This can be overridden by passing one or more
142   * string arguments after the cert name. The expectation for those arguments is that they are
143   * valid DNS names.
144   */
145  public X509Certificate newCert(X500Name name, String... subjectAltNames)
146    throws GeneralSecurityException, IOException, OperatorCreationException {
147    if (subjectAltNames.length == 0) {
148      return X509TestHelpers.newCert(trustStoreCertificate, trustStoreKeyPair, name,
149        keyStoreKeyPair.getPublic());
150    }
151    GeneralName[] names = new GeneralName[subjectAltNames.length];
152    for (int i = 0; i < subjectAltNames.length; i++) {
153      names[i] = new GeneralName(GeneralName.dNSName, subjectAltNames[i]);
154    }
155    return X509TestHelpers.newCert(trustStoreCertificate, trustStoreKeyPair, name,
156      keyStoreKeyPair.getPublic(), new GeneralNames(names));
157  }
158
159  public File getTempDir() {
160    return tempDir;
161  }
162
163  public char[] getTrustStorePassword() {
164    return trustStorePassword;
165  }
166
167  /**
168   * Returns the path to the trust store file in the given format (JKS or PEM). Note that the file
169   * is created lazily, the first time this method is called.
170   * @param storeFileType the store file type (JKS or PEM).
171   * @return the path to the trust store file.
172   * @throws Exception if there is an error creating the trust store file.
173   */
174  public File getTrustStoreFile(KeyStoreFileType storeFileType) throws Exception {
175    switch (storeFileType) {
176      case JKS:
177        return getTrustStoreJksFile();
178      case PEM:
179        return getTrustStorePemFile();
180      case PKCS12:
181        return getTrustStorePkcs12File();
182      case BCFKS:
183        return getTrustStoreBcfksFile();
184      default:
185        throw new IllegalArgumentException("Invalid trust store type: " + storeFileType
186          + ", must be one of: " + Arrays.toString(KeyStoreFileType.values()));
187    }
188  }
189
190  private void atomicWriteFile(File file, byte[] content) throws IOException {
191    File tmpFile = new File(file.getParentFile(), file.getName().concat(".tmp"));
192    Files.write(tmpFile.toPath(), content, StandardOpenOption.CREATE,
193      StandardOpenOption.TRUNCATE_EXISTING);
194    Files.move(tmpFile.toPath(), file.toPath(), StandardCopyOption.REPLACE_EXISTING,
195      StandardCopyOption.ATOMIC_MOVE);
196  }
197
198  private File getTrustStoreJksFile() throws Exception {
199    if (trustStoreJksFile == null) {
200      trustStoreJksFile = File.createTempFile(TRUST_STORE_PREFIX,
201        KeyStoreFileType.JKS.getDefaultFileExtension(), tempDir);
202      generateTrustStoreJksFile();
203    }
204    return trustStoreJksFile;
205  }
206
207  private void generateTrustStoreJksFile() throws Exception {
208    atomicWriteFile(trustStoreJksFile,
209      X509TestHelpers.certToJavaTrustStoreBytes(trustStoreCertificate, trustStorePassword));
210  }
211
212  private File getTrustStorePemFile() throws IOException {
213    if (trustStorePemFile == null) {
214      trustStorePemFile = File.createTempFile(TRUST_STORE_PREFIX,
215        KeyStoreFileType.PEM.getDefaultFileExtension(), tempDir);
216      generateTrustStorePemFile();
217    }
218    return trustStorePemFile;
219  }
220
221  private void generateTrustStorePemFile() throws IOException {
222    atomicWriteFile(trustStorePemFile, X509TestHelpers
223      .pemEncodeX509Certificate(trustStoreCertificate).getBytes(StandardCharsets.US_ASCII));
224  }
225
226  private File getTrustStorePkcs12File() throws Exception {
227    if (trustStorePkcs12File == null) {
228      trustStorePkcs12File = File.createTempFile(TRUST_STORE_PREFIX,
229        KeyStoreFileType.PKCS12.getDefaultFileExtension(), tempDir);
230      generateTrustStorePkcs12File();
231    }
232    return trustStorePkcs12File;
233  }
234
235  private void generateTrustStorePkcs12File() throws Exception {
236    atomicWriteFile(trustStorePkcs12File,
237      X509TestHelpers.certToPKCS12TrustStoreBytes(trustStoreCertificate, trustStorePassword));
238  }
239
240  private File getTrustStoreBcfksFile() throws Exception {
241    if (trustStoreBcfksFile == null) {
242      trustStoreBcfksFile = File.createTempFile(TRUST_STORE_PREFIX,
243        KeyStoreFileType.BCFKS.getDefaultFileExtension(), tempDir);
244      generateTrustStoreBcfksFile();
245    }
246    return trustStoreBcfksFile;
247  }
248
249  private void generateTrustStoreBcfksFile() throws Exception {
250    atomicWriteFile(trustStoreBcfksFile,
251      X509TestHelpers.certToBCFKSTrustStoreBytes(trustStoreCertificate, trustStorePassword));
252  }
253
254  public X509Certificate getKeyStoreCertificate() {
255    return keyStoreCertificate;
256  }
257
258  public char[] getKeyStorePassword() {
259    return keyStorePassword;
260  }
261
262  public boolean isKeyStoreEncrypted() {
263    return keyStorePassword != null;
264  }
265
266  public Configuration getConf() {
267    return conf;
268  }
269
270  /**
271   * Returns the path to the key store file in the given format (JKS, PEM, ...). Note that the file
272   * is created lazily, the first time this method is called.
273   * @param storeFileType the store file type (JKS, PEM, ...).
274   * @return the path to the key store file.
275   * @throws Exception if there is an error creating the key store file.
276   */
277  public File getKeyStoreFile(KeyStoreFileType storeFileType) throws Exception {
278    switch (storeFileType) {
279      case JKS:
280        return getKeyStoreJksFile();
281      case PEM:
282        return getKeyStorePemFile();
283      case PKCS12:
284        return getKeyStorePkcs12File();
285      case BCFKS:
286        return getKeyStoreBcfksFile();
287      default:
288        throw new IllegalArgumentException("Invalid key store type: " + storeFileType
289          + ", must be one of: " + Arrays.toString(KeyStoreFileType.values()));
290    }
291  }
292
293  private File getKeyStoreJksFile() throws Exception {
294    if (keyStoreJksFile == null) {
295      keyStoreJksFile = File.createTempFile(KEY_STORE_PREFIX,
296        KeyStoreFileType.JKS.getDefaultFileExtension(), tempDir);
297      generateKeyStoreJksFile();
298    }
299    return keyStoreJksFile;
300  }
301
302  private void generateKeyStoreJksFile() throws Exception {
303    atomicWriteFile(keyStoreJksFile, X509TestHelpers.certAndPrivateKeyToJavaKeyStoreBytes(
304      keyStoreCertificate, keyStoreKeyPair.getPrivate(), keyStorePassword));
305  }
306
307  private File getKeyStorePemFile() throws Exception {
308    if (keyStorePemFile == null) {
309      keyStorePemFile = File.createTempFile(KEY_STORE_PREFIX,
310        KeyStoreFileType.PEM.getDefaultFileExtension(), tempDir);
311      generateKeyStorePemFile();
312    }
313    return keyStorePemFile;
314  }
315
316  private void generateKeyStorePemFile() throws Exception {
317    atomicWriteFile(keyStorePemFile, X509TestHelpers.pemEncodeCertAndPrivateKey(keyStoreCertificate,
318      keyStoreKeyPair.getPrivate(), keyStorePassword).getBytes(StandardCharsets.US_ASCII));
319  }
320
321  private File getKeyStorePkcs12File() throws Exception {
322    if (keyStorePkcs12File == null) {
323      keyStorePkcs12File = File.createTempFile(KEY_STORE_PREFIX,
324        KeyStoreFileType.PKCS12.getDefaultFileExtension(), tempDir);
325      generateKeyStorePkcs12File();
326    }
327    return keyStorePkcs12File;
328  }
329
330  private void generateKeyStorePkcs12File() throws Exception {
331    atomicWriteFile(keyStorePkcs12File, X509TestHelpers.certAndPrivateKeyToPKCS12Bytes(
332      keyStoreCertificate, keyStoreKeyPair.getPrivate(), keyStorePassword));
333  }
334
335  private File getKeyStoreBcfksFile() throws Exception {
336    if (keyStoreBcfksFile == null) {
337      keyStoreBcfksFile = File.createTempFile(KEY_STORE_PREFIX,
338        KeyStoreFileType.BCFKS.getDefaultFileExtension(), tempDir);
339      generateKeyStoreBcfksFile();
340    }
341    return keyStoreBcfksFile;
342  }
343
344  private void generateKeyStoreBcfksFile() throws Exception {
345    atomicWriteFile(keyStoreBcfksFile, X509TestHelpers.certAndPrivateKeyToBCFKSBytes(
346      keyStoreCertificate, keyStoreKeyPair.getPrivate(), keyStorePassword));
347  }
348
349  /**
350   * Sets the SSL system properties such that the given X509Util object can be used to create SSL
351   * Contexts that will use the trust store and key store files created by this test context.
352   * Example usage:
353   *
354   * <pre>
355   *     X509TestContext testContext = ...; // create the test context
356   *     X509Util x509Util = new QuorumX509Util();
357   *     testContext.setSystemProperties(x509Util, KeyStoreFileType.JKS, KeyStoreFileType.JKS);
358   *     // The returned context will use the key store and trust store created by the test context.
359   *     SSLContext ctx = x509Util.getDefaultSSLContext();
360   * </pre>
361   *
362   * @param keyStoreFileType   the store file type to use for the key store (JKS, PEM, ...).
363   * @param trustStoreFileType the store file type to use for the trust store (JKS, PEM, ...).
364   * @throws Exception if there is an error creating the key store file or trust store file.
365   */
366  public void setConfigurations(KeyStoreFileType keyStoreFileType,
367    KeyStoreFileType trustStoreFileType) throws Exception {
368    setKeystoreConfigurations(keyStoreFileType, conf);
369    conf.set(X509Util.TLS_CONFIG_TRUSTSTORE_LOCATION,
370      this.getTrustStoreFile(trustStoreFileType).getAbsolutePath());
371    conf.set(X509Util.TLS_CONFIG_TRUSTSTORE_PASSWORD, String.valueOf(this.getTrustStorePassword()));
372    conf.set(X509Util.TLS_CONFIG_TRUSTSTORE_TYPE, trustStoreFileType.getPropertyValue());
373  }
374
375  /**
376   * Sets the KeyStore-related SSL system properties onto the given Configuration such that X509Util
377   * can be used to create SSL Contexts using that KeyStore. This can be used in special
378   * circumstances to inject a "bad" certificate where the keystore doesn't match the CA in the
379   * truststore. Or use it to create a connection without a truststore.
380   * @see #setConfigurations(KeyStoreFileType, KeyStoreFileType) which sets both keystore and
381   *      truststore and is more applicable to general use.
382   */
383  public void setKeystoreConfigurations(KeyStoreFileType keyStoreFileType, Configuration confToSet)
384    throws Exception {
385
386    confToSet.set(X509Util.TLS_CONFIG_KEYSTORE_LOCATION,
387      this.getKeyStoreFile(keyStoreFileType).getAbsolutePath());
388    confToSet.set(X509Util.TLS_CONFIG_KEYSTORE_PASSWORD,
389      String.valueOf(this.getKeyStorePassword()));
390    confToSet.set(X509Util.TLS_CONFIG_KEYSTORE_TYPE, keyStoreFileType.getPropertyValue());
391  }
392
393  public void clearConfigurations() {
394    conf.unset(X509Util.TLS_CONFIG_KEYSTORE_LOCATION);
395    conf.unset(X509Util.TLS_CONFIG_KEYSTORE_PASSWORD);
396    conf.unset(X509Util.TLS_CONFIG_KEYSTORE_TYPE);
397    conf.unset(X509Util.TLS_CONFIG_TRUSTSTORE_LOCATION);
398    conf.unset(X509Util.TLS_CONFIG_TRUSTSTORE_PASSWORD);
399    conf.unset(X509Util.TLS_CONFIG_TRUSTSTORE_TYPE);
400  }
401
402  /**
403   * Creates a clone of the current context, but injecting the passed certificate as the KeyStore
404   * cert. The new context's keystore path fields are nulled, so the next call to
405   * {@link #setConfigurations(KeyStoreFileType, KeyStoreFileType)},
406   * {@link #setKeystoreConfigurations(KeyStoreFileType, Configuration)} , or
407   * {@link #getKeyStoreFile(KeyStoreFileType)} will create a new keystore with this certificate in
408   * place.
409   * @param cert the cert to replace
410   */
411  public X509TestContext cloneWithNewKeystoreCert(X509Certificate cert) {
412    return new X509TestContext(tempDir, conf, trustStoreCertificate, trustStorePassword,
413      trustStoreKeyPair, trustStoreJksFile, trustStorePemFile, trustStorePkcs12File,
414      keyStoreKeyPair, keyStorePassword, cert);
415  }
416
417  public void regenerateStores(X509KeyType keyStoreKeyType, X509KeyType trustStoreKeyType,
418    KeyStoreFileType keyStoreFileType, KeyStoreFileType trustStoreFileType,
419    String... subjectAltNames) throws Exception {
420    trustStoreKeyPair = X509TestHelpers.generateKeyPair(trustStoreKeyType);
421    keyStoreKeyPair = X509TestHelpers.generateKeyPair(keyStoreKeyType);
422    createCertificates(subjectAltNames);
423
424    switch (keyStoreFileType) {
425      case JKS:
426        generateKeyStoreJksFile();
427        break;
428      case PEM:
429        generateKeyStorePemFile();
430        break;
431      case BCFKS:
432        generateKeyStoreBcfksFile();
433        break;
434      case PKCS12:
435        generateKeyStorePkcs12File();
436        break;
437    }
438
439    switch (trustStoreFileType) {
440      case JKS:
441        generateTrustStoreJksFile();
442        break;
443      case PEM:
444        generateTrustStorePemFile();
445        break;
446      case PKCS12:
447        generateTrustStorePkcs12File();
448        break;
449      case BCFKS:
450        generateTrustStoreBcfksFile();
451        break;
452    }
453  }
454
455  private void createCertificates(String... subjectAltNames)
456    throws GeneralSecurityException, IOException, OperatorCreationException {
457    X500NameBuilder caNameBuilder = new X500NameBuilder(BCStyle.INSTANCE);
458    caNameBuilder.addRDN(BCStyle.CN,
459      MethodHandles.lookup().lookupClass().getCanonicalName() + " Root CA");
460    trustStoreCertificate =
461      X509TestHelpers.newSelfSignedCACert(caNameBuilder.build(), trustStoreKeyPair);
462
463    X500NameBuilder nameBuilder = new X500NameBuilder(BCStyle.INSTANCE);
464    nameBuilder.addRDN(BCStyle.CN,
465      MethodHandles.lookup().lookupClass().getCanonicalName() + " Zookeeper Test");
466    keyStoreCertificate = newCert(nameBuilder.build(), subjectAltNames);
467  }
468
469  /**
470   * Builder class, used for creating new instances of X509TestContext.
471   */
472  public static class Builder {
473
474    private final Configuration conf;
475    private File tempDir;
476    private X509KeyType trustStoreKeyType;
477    private char[] trustStorePassword;
478    private X509KeyType keyStoreKeyType;
479    private char[] keyStorePassword;
480
481    /**
482     * Creates an empty builder with the given Configuration.
483     */
484    public Builder(Configuration conf) {
485      this.conf = conf;
486      trustStoreKeyType = X509KeyType.EC;
487      keyStoreKeyType = X509KeyType.EC;
488    }
489
490    /**
491     * Builds a new X509TestContext from this builder.
492     * @return a new X509TestContext
493     */
494    public X509TestContext build()
495      throws IOException, GeneralSecurityException, OperatorCreationException {
496      KeyPair trustStoreKeyPair = X509TestHelpers.generateKeyPair(trustStoreKeyType);
497      KeyPair keyStoreKeyPair = X509TestHelpers.generateKeyPair(keyStoreKeyType);
498      return new X509TestContext(conf, tempDir, trustStoreKeyPair, trustStorePassword,
499        keyStoreKeyPair, keyStorePassword);
500    }
501
502    /**
503     * Sets the temporary directory. Certificate and private key files will be created in this
504     * directory.
505     * @param tempDir the temp directory.
506     * @return this Builder.
507     */
508    public Builder setTempDir(File tempDir) {
509      this.tempDir = tempDir;
510      return this;
511    }
512
513    /**
514     * Sets the trust store key type. The CA key generated for the test context will be of this
515     * type.
516     * @param keyType the key type.
517     * @return this Builder.
518     */
519    public Builder setTrustStoreKeyType(X509KeyType keyType) {
520      trustStoreKeyType = keyType;
521      return this;
522    }
523
524    /**
525     * Sets the trust store password. Ignored for PEM trust stores, JKS trust stores will be
526     * encrypted with this password.
527     * @param password the password.
528     * @return this Builder.
529     */
530    public Builder setTrustStorePassword(char[] password) {
531      trustStorePassword = password;
532      return this;
533    }
534
535    /**
536     * Sets the key store key type. The private key generated for the test context will be of this
537     * type.
538     * @param keyType the key type.
539     * @return this Builder.
540     */
541    public Builder setKeyStoreKeyType(X509KeyType keyType) {
542      keyStoreKeyType = keyType;
543      return this;
544    }
545
546    /**
547     * Sets the key store password. The private key (PEM, JKS) and certificate (JKS only) will be
548     * encrypted with this password.
549     * @param password the password.
550     * @return this Builder.
551     */
552    public Builder setKeyStorePassword(char[] password) {
553      keyStorePassword = password;
554      return this;
555    }
556  }
557
558  /**
559   * Returns a new default-constructed Builder.
560   * @return a new Builder.
561   */
562  public static Builder newBuilder(Configuration conf) {
563    return new Builder(conf);
564  }
565}