This is a prerelease version.

View latest

Persistence Configuration Options

Explore the options for configuring persistence on your members.

Global Persistence Configuration

Use the following options to configure the persistence object on your members.

  • XML

  • YAML

  • Java

<hazelcast>
  <persistence enabled="true">
    <!-- insert global configuration options here -->
  </persistence>
</hazelcast>
hazelcast:
  persistence:
    enabled: true
    # insert global configuration options here

Add global configuration options to the PersistenceConfig object.

Config config = new Config();

PersistenceConfig PersistenceConfig = new PersistenceConfig()
.setEnabled(true);

config.setPersistenceConfig(PersistenceConfig);
Option Description Default Example

base-dir

The parent directory in which to store persisted data.

This directory is created automatically if it does not exist.

persistence

  • XML

  • YAML

  • Java

<hazelcast>
  <persistence enabled="true">
    <base-dir>
      /path/to/persistence
    </base-dir>
  </persistence>
</hazelcast>
hazelcast:
  persistence:
    enabled: true
    base-dir: /path/to/persistence
Config config = new Config();

PersistenceConfig PersistenceConfig = new PersistenceConfig()
.setEnabled(true)
.setBaseDir(new File("/path/to/persistence"));

config.setPersistenceConfig(PersistenceConfig);

backup-dir

The directory in which to store backup snapshots (hot backups).

' ' (empty)

parallelism

Number of I/O threads that are started concurrently for reading from and writing to files in the persistence store.

Before changing the default, you should measure the raw I/O throughput of your infrastructure and test with different values of parallelism. In some cases, such as dedicated hardware, higher parallelism can yield more throughput. In other cases, such as running on EC2, a higher parallelism can yield diminishing returns with more thread scheduling, more contention on I/O, and less efficient garbage collection.

1

  • XML

  • YAML

  • Java

<hazelcast>
  <persistence enabled="true">
    <parallelism>
      2
    </parallelism>
  </persistence>
</hazelcast>
hazelcast:
  persistence:
    enabled: true
    parallelism: 2
Config config = new Config();

PersistenceConfig PersistenceConfig = new PersistenceConfig()
.setEnabled(true)
.setParallelism(2);

config.setPersistenceConfig(PersistenceConfig);

validation-timeout-seconds

Number of seconds that the cluster allows for members to rejoin and send their partition table to the master member.

120

  • XML

  • YAML

  • Java

<hazelcast>
  <persistence enabled="true">
    <validation-timeout-seconds>
      120
    </validation-timeout-seconds>
  </persistence>
</hazelcast>
hazelcast:
  persistence:
    enabled: true
    validation-timeout-seconds: 120
Config config = new Config();

PersistenceConfig PersistenceConfig = new PersistenceConfig()
.setEnabled(true)
.setValidationTimeoutSeconds(120);

config.setPersistenceConfig(PersistenceConfig);

data-load-timeout-seconds

Number of seconds that the cluster allows for members to finish restoring data from their local persistence store.

900

  • XML

  • YAML

  • Java

<hazelcast>
  <persistence enabled="true">
    <data-load-timeout-seconds>
      900
    </data-load-timeout-seconds>
  </persistence>
</hazelcast>
hazelcast:
  persistence:
    enabled: true
    data-load-timeout-seconds: 900
Config config = new Config();

PersistenceConfig PersistenceConfig = new PersistenceConfig()
.setEnabled(true)
.setDataLoadTimeoutSeconds(900);

config.setPersistenceConfig(PersistenceConfig);

cluster-data-recovery-policy

The data recovery policy that is respected when the whole cluster restarts.

Valid values are:

  • FULL_RECOVERY_ONLY: Starts the cluster only when all expected members are present and correct.

  • PARTIAL_RECOVERY_MOST_RECENT: Starts the cluster with the members that have most up-to-date partition table and successfully restored their data. All other members leave the cluster and force start themselves. If no members restore their data successfully, the cluster start fails.

  • PARTIAL_RECOVERY_MOST_COMPLETE: Starts the cluster with the largest group of members that have the same partition table version and successfully restored their data. All other members leave the cluster and force start themselves. If no members restore their data successfully, the cluster start fails.

FULL_RECOVERY_ONLY

  • XML

  • YAML

  • Java

<hazelcast>
  <persistence enabled="true">
    <cluster-data-recovery-policy>
      FULL_RECOVERY_ONLY
    </cluster-data-recovery-policy>
  </persistence>
</hazelcast>
hazelcast:
  persistence:
    enabled: true
    cluster-data-recovery-policy: FULL_RECOVERY_ONLY
Config config = new Config();

PersistenceConfig PersistenceConfig = new PersistenceConfig()
.setEnabled(true)
.setClusterDataRecoveryPolicy(PersistenceClusterDataRecoveryPolicy.FULL_RECOVERY_ONLY);

config.setPersistenceConfig(PersistenceConfig);

auto-remove-stale-data

Enables a joining member to automatically remove its persistence store if the cluster’s master member considers the persisted data to be stale. See Handling Member Failures with Persistence Enabled.

true

  • XML

  • YAML

  • Java

<hazelcast>
  <persistence enabled="true">
    <auto-remove-stale-data>
      true
    </auto-remove-stale-data>
  </persistence>
</hazelcast>
hazelcast:
  persistence:
    enabled: true
    auto-remove-stale-data: true
Config config = new Config();

PersistenceConfig PersistenceConfig = new PersistenceConfig()
.setEnabled(true)
.setAutoRemoveStaleData(true);

config.setPersistenceConfig(PersistenceConfig);

encryption-at-rest

Enables encryption of data in the persistence store.

disabled

rebalance-delay-seconds

Number of seconds a cluster waits before repartitioning after a member leaves by means other than a graceful shutdown.

0

Data Structure Persistence Configuration

Use the following options to configure persistence for map and JCache data structures.

  • XML

  • YAML

  • Java

<hazelcast>
  <map name="test-map">
    <data-persistence enabled="true">
      <!-- insert configuration options here -->
    </data-persistence>
  </map>
</hazelcast>
hazelcast:
  map:
  test-map:
    data-persistence:
      enabled: true
      # insert configuration options here

Add configuration options to the MapConfig object.

Config config = new Config();

MapConfig mapConfig = config.getMapConfig("test-map");
mapConfig.getDataPersistenceConfig().setEnabled(true);

config.addMapConfig(mapConfig);
Option Description Default Example

fsync

Guarantees that data is persisted to disk when a write operation returns a successful response to the caller.

By default, data is eventually persisted to disk instead of on every disk write. This generally provides a better performance.

false

  • XML

  • YAML

  • Java

<hazelcast>
  <map name="test-map">
    <data-persistence enabled="true">
      <fsync>
        false
      </fsync>
    </data-persistence>
  </map>
</hazelcast>
hazelcast:
  map:
  test-map:
    data-persistence:
      enabled: true
      fsync: false
Config config = new Config();

MapConfig mapConfig = config.getMapConfig("test-map");
mapConfig.getDataPersistenceConfig().setEnabled(true)
.setFsync(true);

config.addMapConfig(mapConfig);

merkle-tree

Allows restarting members to synchronize their persisted map or JCache data faster with the rest of the cluster.

enabled

See Synchronizing Data Faster for more information.

Job Snapshot Configuration

Use the following option to configure persistence for job snapshots.

  • XML

  • YAML

  • Java

<hazelcast>
  <jet>
    <instance>
      <lossless-restart-enabled>
      true
      </lossless-restart-enabled>
    </instance>
  </jet>
</hazelcast>
hazelcast:
  jet:
    instance:
      lossless-restart-enabled: true

Use the JetConfig object.

Config config = new Config();
config.getJetConfig().setLosslessRestartEnabled(true);

For lossless restart to work, the cluster must be shut down gracefully. When members are shut down in a rapid succession, Hazelcast triggers an automatic rebalancing process where backup partitions are promoted and new backups are created for each member. This may result in out-of-memory errors or data loss.

Because job data is saved locally on each member, all members must be present after a restart for Hazelcast to be able to reload the data.

Full Example of Persistence Configuration

The following are example configuration settings for a map instance, a JCache instance, and the Jet engine.

  • XML

  • YAML

  • Java

<hazelcast>
    ...
    <persistence enabled="true">
      <base-dir>/mnt/persistence</base-dir>
      <backup-dir>/mnt/hot-backup</backup-dir>
      <validation-timeout-seconds>120</validation-timeout-seconds>
      <data-load-timeout-seconds>900</data-load-timeout-seconds>
      <cluster-data-recovery-policy>FULL_RECOVERY_ONLY</cluster-data-recovery-policy>
      <rebalance-delay-seconds>0</rebalance-delay-seconds>
    </persistence>
    ...
    <map name="test-map">
      <merkle-tree enabled="true" >
        <depth>12</depth>
      </merkle-tree>
      <data-persistence enabled="true">
        <fsync>false</fsync>
      </data-persistence>
    </map>
    ...
    <cache name="test-cache">
      <merkle-tree enabled="true" >
        <depth>12</depth>
      </merkle-tree>
      <data-persistence enabled="true">
          <fsync>false</fsync>
      </data-persistence>
    </cache>
    ...
    <jet>
      <instance>
        <lossless-restart-enabled>true</lossless-restart-enabled>
      </instance>
    </jet>
    ...
</hazelcast>
hazelcast:
  persistence:
    enabled: true
    base-dir: /mnt/persistence
    backup-dir: /mnt/hot-backup
    validation-timeout-seconds: 120
    data-load-timeout-seconds: 900
    cluster-data-recovery-policy: FULL_RECOVERY_ONLY
    rebalance-delay-seconds: 0
  map:
    test-map:
      merkle-tree:
        enabled: true
        depth: 12
      data-persistence:
        enabled: true
        fsync: false
  cache:
    test-cache:
      merkle-tree:
        enabled: true
        depth: 12
      data-persistence:
        enabled: true
        fsync: false
  jet:
    instance:
      lossless-restart-enabled: true
        Config config = new Config();
        PersistenceConfig PersistenceConfig = new PersistenceConfig()
        .setEnabled(true)
        .setBaseDir(new File("/mnt/persistence"))
        .setParallelism(1)
        .setValidationTimeoutSeconds(120)
        .setDataLoadTimeoutSeconds(900)
        .setClusterDataRecoveryPolicy(PersistenceClusterDataRecoveryPolicy.FULL_RECOVERY_ONLY)
        .setAutoRemoveStaleData(true);
        .setRebalanceDelaySeconds(0)
        config.setPersistenceConfig(PersistenceConfig);

        MapConfig mapConfig = config.getMapConfig("test-map");
        mapConfig.getDataPersistenceConfig().setEnabled(true);
        mapConfig.getMerkleTreeConfig().setEnabled(true);
        mapConfig.getMerkleTreeConfig().setDepth(12);
        config.addMapConfig(mapConfig);

        CacheSimpleConfig cacheConfig = config.getCacheConfig("test-cache");
        cacheConfig.getDataPersistenceConfig().setEnabled(true);
        cacheConfig.getMerkleTreeConfig().setEnabled(true);
        cacheConfig.getMerkleTreeConfig().setDepth(12);
        config.addCacheConfig(cacheConfig);

        config.getJetConfig().setLosslessRestartEnabled(true);