Setting Hazelcast Cache for Multi-tenancy

Question

I am currently using the JHipster generator for really boiler plate code which involves HazelCast as a second level cache. I was able to get Multi-tenancy (schema per tenant) working with a header based tenant context. The problem I have now, is that the @Cacheable annotations all share a context. If the cache is hot, I end up with cross-schema data. For example, tenant1 pulls all records from their table which is cached. Tenant 2 goes to pull the same items from their table, the cache is read, and it never goes to the actual tenant db. An easy fix would be disable caching all together but I would like to not do that. I can not for the life of me figure out how to make hazelcast aware of the tenant context - documentation is lacking. Some others have solved this with using custom name resolvers but it doesn't appear to be as dynamic as I was hoping (i.e. you have to know all of the tenants ahead of time). Thoughts?

Current cache config:

@Configuration
@EnableCaching
public class CacheConfiguration implements DisposableBean {

    private final Logger log = LoggerFactory.getLogger(CacheConfiguration.class);

    private final Environment env;

    private final ServerProperties serverProperties;

    private final DiscoveryClient discoveryClient;

    private Registration registration;

    public CacheConfiguration(Environment env, ServerProperties serverProperties, DiscoveryClient discoveryClient) {
        this.env = env;
        this.serverProperties = serverProperties;
        this.discoveryClient = discoveryClient;
    }

    @Autowired(required = false)
    public void setRegistration(Registration registration) {
        this.registration = registration;
    }

    @Override
    public void destroy() throws Exception {
        log.info("Closing Cache Manager");
        Hazelcast.shutdownAll();
    }

    @Bean
    public CacheManager cacheManager(HazelcastInstance hazelcastInstance) {
        log.debug("Starting HazelcastCacheManager");
        return new com.hazelcast.spring.cache.HazelcastCacheManager(hazelcastInstance);
    }

    @Bean
    public HazelcastInstance hazelcastInstance(JHipsterProperties jHipsterProperties) {
        log.debug("Configuring Hazelcast");
        HazelcastInstance hazelCastInstance = Hazelcast.getHazelcastInstanceByName("SampleApp");
        if (hazelCastInstance != null) {
            log.debug("Hazelcast already initialized");
            return hazelCastInstance;
        }
        Config config = new Config();
        config.setInstanceName("SampleApp");
        config.getNetworkConfig().getJoin().getMulticastConfig().setEnabled(false);
        if (this.registration == null) {
            log.warn("No discovery service is set up, Hazelcast cannot create a cluster.");
        } else {
            // The serviceId is by default the application's name,
            // see the "spring.application.name" standard Spring property
            String serviceId = registration.getServiceId();
            log.debug("Configuring Hazelcast clustering for instanceId: {}", serviceId);
            // In development, everything goes through 127.0.0.1, with a different port
            if (env.acceptsProfiles(Profiles.of(JHipsterConstants.SPRING_PROFILE_DEVELOPMENT))) {
                log.debug("Application is running with the \"dev\" profile, Hazelcast " +
                          "cluster will only work with localhost instances");

                System.setProperty("hazelcast.local.localAddress", "127.0.0.1");
                config.getNetworkConfig().setPort(serverProperties.getPort() + 5701);
                config.getNetworkConfig().getJoin().getTcpIpConfig().setEnabled(true);
                for (ServiceInstance instance : discoveryClient.getInstances(serviceId)) {
                    String clusterMember = "127.0.0.1:" + (instance.getPort() + 5701);
                    log.debug("Adding Hazelcast (dev) cluster member {}", clusterMember);
                    config.getNetworkConfig().getJoin().getTcpIpConfig().addMember(clusterMember);
                }
            } else { // Production configuration, one host per instance all using port 5701
                config.getNetworkConfig().setPort(5701);
                config.getNetworkConfig().getJoin().getTcpIpConfig().setEnabled(true);
                for (ServiceInstance instance : discoveryClient.getInstances(serviceId)) {
                    String clusterMember = instance.getHost() + ":5701";
                    log.debug("Adding Hazelcast (prod) cluster member {}", clusterMember);
                    config.getNetworkConfig().getJoin().getTcpIpConfig().addMember(clusterMember);
                }
            }
        }
        config.getMapConfigs().put("default", initializeDefaultMapConfig(jHipsterProperties));

        // Full reference is available at: http://docs.hazelcast.org/docs/management-center/3.9/manual/html/Deploying_and_Starting.html
        config.setManagementCenterConfig(initializeDefaultManagementCenterConfig(jHipsterProperties));
        config.getMapConfigs().put("com.test.sampleapp.domain.*", initializeDomainMapConfig(jHipsterProperties));
        return Hazelcast.newHazelcastInstance(config);
    }

    private ManagementCenterConfig initializeDefaultManagementCenterConfig(JHipsterProperties jHipsterProperties) {
        ManagementCenterConfig managementCenterConfig = new ManagementCenterConfig();
        managementCenterConfig.setEnabled(jHipsterProperties.getCache().getHazelcast().getManagementCenter().isEnabled());
        managementCenterConfig.setUrl(jHipsterProperties.getCache().getHazelcast().getManagementCenter().getUrl());
        managementCenterConfig.setUpdateInterval(jHipsterProperties.getCache().getHazelcast().getManagementCenter().getUpdateInterval());
        return managementCenterConfig;
    }

    private MapConfig initializeDefaultMapConfig(JHipsterProperties jHipsterProperties) {
        MapConfig mapConfig = new MapConfig();

        /*
        Number of backups. If 1 is set as the backup-count for example,
        then all entries of the map will be copied to another JVM for
        fail-safety. Valid numbers are 0 (no backup), 1, 2, 3.
        */
        mapConfig.setBackupCount(jHipsterProperties.getCache().getHazelcast().getBackupCount());

        /*
        Valid values are:
        NONE (no eviction),
        LRU (Least Recently Used),
        LFU (Least Frequently Used).
        NONE is the default.
        */
        mapConfig.setEvictionPolicy(EvictionPolicy.LRU);

        /*
        Maximum size of the map. When max size is reached,
        map is evicted based on the policy defined.
        Any integer between 0 and Integer.MAX_VALUE. 0 means
        Integer.MAX_VALUE. Default is 0.
        */
        mapConfig.setMaxSizeConfig(new MaxSizeConfig(0, MaxSizeConfig.MaxSizePolicy.USED_HEAP_SIZE));

        return mapConfig;
    }

    private MapConfig initializeDomainMapConfig(JHipsterProperties jHipsterProperties) {
        MapConfig mapConfig = new MapConfig();
        mapConfig.setTimeToLiveSeconds(jHipsterProperties.getCache().getHazelcast().getTimeToLiveSeconds());
        return mapConfig;
    }
}

Sample Repository using cacheNames...

@Repository
public interface UserRepository extends JpaRepository<User, Long> {

    String USERS_BY_LOGIN_CACHE = "usersByLogin";

    String USERS_BY_EMAIL_CACHE = "usersByEmail";

    String USERS_BY_ID_CACHE = "usersById";

    Optional<User> findOneByActivationKey(String activationKey);

    List<User> findAllByActivatedIsFalseAndActivationKeyIsNotNullAndCreatedDateBefore(Instant dateTime);

    Optional<User> findOneByResetKey(String resetKey);

    Optional<User> findOneByEmailIgnoreCase(String email);

    Optional<User> findOneByLogin(String login);

    @EntityGraph(attributePaths = "roles")
    @Cacheable(cacheNames = USERS_BY_ID_CACHE)
    Optional<User> findOneWithRolesById(Long id);

    @EntityGraph(attributePaths = "roles")
    @Cacheable(cacheNames = USERS_BY_LOGIN_CACHE)
    Optional<User> findOneWithRolesByLogin(String login);

    @EntityGraph(attributePaths = { "roles", "roles.permissions" })
    @Cacheable(cacheNames = USERS_BY_LOGIN_CACHE)
    Optional<User> findOneWithRolesAndPermissionsByLogin(String login);

    @EntityGraph(attributePaths = "roles")
    @Cacheable(cacheNames = USERS_BY_EMAIL_CACHE)
    Optional<User> findOneWithRolesByEmail(String email);

    Page<User> findAllByLoginNot(Pageable pageable, String login);
}

Answer 1

I am using tenant per database (MySQL), but as long as you are setting a thread context above here is what I'm doing - I'm using Spring Boot. I've created a custom Cache Key generator which combines the tenant name + class + and method. You can really choose any combination. Whenever I pass that tenant back it pulls the correct entries. In the Hazelcast command center for my AppointmentType map type I see the number of entries increment per tenant.

Some other references that may be helpful:

• https://www.javadevjournal.com/spring/spring-cache-custom-keygenerator/
• https://docs.spring.io/spring-framework/docs/4.3.x/spring-framework-reference/html/cache.html (search for keyGenerator="myKeyGenerator")

In your class where you want to cache (mine is a service class):

@Service
public class AppointmentTypeService {

    private static final Logger LOGGER = LoggerFactory.getLogger(AppointmentTypeService.class);

    private final AppointmentTypeRepository appointmentTypeRepository;

    @Autowired
    AppointmentTypeService(AppointmentTypeRepository appointmentTypeRepository) {
        this.appointmentTypeRepository = appointmentTypeRepository;
    }
    
    //ADD keyGenerator value. Name is the name of the bean of the class
    @Cacheable(value="appointmentType", keyGenerator = "multiTenantCacheKeyGenerator")
    public List<AppointmentType> list() {
        return this.appointmentTypeRepository.findAll();
    }

    @CacheEvict(value="appointmentType", allEntries=true)
    public Long create(AppointmentType request) {
        this.appointmentTypeRepository.saveAndFlush(request);
        return request.getAppointmentTypeId();
    }

    @CacheEvict(value="appointmentType", allEntries=true)
    public void delete(Long id) {
        this.appointmentTypeRepository.deleteById(id);
    }

    public Optional<AppointmentType> findById(Long id) {
        return this.appointmentTypeRepository.findById(id);
    }

}

Create key generator class

//setting the bean name here
@Component("multiTenantCacheKeyGenerator")
public class MultiTenantCacheKeyGenerator implements KeyGenerator {

    @Override
    public Object generate(Object o, Method method, Object... os) {
        StringBuilder sb = new StringBuilder();
        sb.append(TenantContext.getCurrentTenantInstanceName()) //my tenant context class which is using local thread. I set the value in the Spring filter.
          .append("_")
          .append(o.getClass().getSimpleName())
          .append("-")
          .append(method.getName());
      }

       return sb.toString();
    }
    
}