Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

adjust recipe input expected cache size dynamically #2331

Merged
merged 4 commits into from
Apr 4, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/main/java/gregtech/GregTechMod.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import gregtech.api.GTValues;
import gregtech.api.GregTechAPI;
import gregtech.api.modules.ModuleContainerRegistryEvent;
import gregtech.api.persistence.PersistentData;
import gregtech.client.utils.BloomEffectUtil;
import gregtech.modules.GregTechModules;
import gregtech.modules.ModuleManager;
Expand Down Expand Up @@ -50,6 +51,7 @@ public GregTechMod() {

@EventHandler
public void onConstruction(FMLConstructionEvent event) {
PersistentData.instance().init();
moduleManager = ModuleManager.getInstance();
GregTechAPI.moduleManager = moduleManager;
moduleManager.registerContainer(new GregTechModules());
Expand Down
108 changes: 108 additions & 0 deletions src/main/java/gregtech/api/persistence/PersistentData.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
package gregtech.api.persistence;

import gregtech.api.GTValues;
import gregtech.api.util.GTLog;

import net.minecraft.nbt.CompressedStreamTools;
import net.minecraft.nbt.NBTTagCompound;
import net.minecraftforge.fml.common.Loader;

import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;

public final class PersistentData {

private static final PersistentData INSTANCE = new PersistentData();

private @Nullable Path path;
private @Nullable NBTTagCompound tag;

public static @NotNull PersistentData instance() {
return INSTANCE;
}

private PersistentData() {}

@ApiStatus.Internal
public void init() {
this.path = Loader.instance().getConfigDir().toPath()
.resolve(GTValues.MODID)
.resolve("persistent_data.dat");
}

/**
* @return the stored persistent data
*/
public synchronized @NotNull NBTTagCompound getTag() {
if (this.tag == null) {
this.tag = read();
}
return this.tag;
}

/**
* @return the read NBTTagCompound from disk
*/
private @NotNull NBTTagCompound read() {
GTLog.logger.debug("Reading persistent data from path {}", path);
if (this.path == null) {
throw new IllegalStateException("Persistent data path cannot be null");
}

if (!Files.exists(path)) {
return new NBTTagCompound();
}

try (InputStream inputStream = Files.newInputStream(this.path)) {
return CompressedStreamTools.readCompressed(inputStream);
} catch (IOException e) {
GTLog.logger.error("Failed to read persistent data", e);
return new NBTTagCompound();
}
}

/**
* Save the GT Persistent data to disk
*/
public synchronized void save() {
if (this.tag != null) {
write(this.tag);
}
}

/**
* @param tagCompound the tag compound to save to disk
*/
private void write(@NotNull NBTTagCompound tagCompound) {
GTLog.logger.debug("Writing persistent data to path {}", path);
if (tagCompound.isEmpty()) {
return;
}

if (this.path == null) {
throw new IllegalStateException("Persistent data path cannot be null");
}

if (!Files.exists(path)) {
try {
Files.createDirectories(path.getParent());
} catch (IOException e) {
GTLog.logger.error("Could not create persistent data dir", e);
return;
}
}

try (OutputStream outputStream = Files.newOutputStream(path)) {
CompressedStreamTools.writeCompressed(tagCompound, outputStream);
} catch (IOException e) {
GTLog.logger.error("Failed to write persistent data", e);
}
}
}
124 changes: 100 additions & 24 deletions src/main/java/gregtech/api/recipes/GTRecipeInputCache.java
Original file line number Diff line number Diff line change
@@ -1,11 +1,18 @@
package gregtech.api.recipes;

import gregtech.api.GTValues;
import gregtech.api.persistence.PersistentData;
import gregtech.api.recipes.ingredients.GTRecipeInput;
import gregtech.api.util.GTLog;
import gregtech.common.ConfigHolder;

import net.minecraft.nbt.NBTTagCompound;
import net.minecraft.util.math.MathHelper;

import it.unimi.dsi.fastutil.Hash;
import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;

import java.util.ArrayList;
import java.util.Collections;
Expand All @@ -14,64 +21,87 @@
/**
* Cache of GTRecipeInput instances for deduplication.
* <p>
* Each GTRecipeInput is cached by an internal hashtable, and any duplicative
* instances will be replaced by identical object previously created.
* Each GTRecipeInput is cached by an internal hashtable, and any duplicative instances will be replaced by identical
* object previously created.
* <p>
* Caching and duplication is only available during recipe registration; once
* recipe registration is over, the cache will be discarded and no further entries
* will be put into cache.
* Caching and duplication is only available during recipe registration; once recipe registration is over, the cache
* will be discarded and no further entries will be put into cache.
*/
public class GTRecipeInputCache {
public final class GTRecipeInputCache {

private static final int MINIMUM_CACHE_SIZE = 1 << 13;
private static final int MAXIMUM_CACHE_SIZE = 1 << 30;

private static ObjectOpenHashSet<GTRecipeInput> instances;

private static final int EXPECTED_CACHE_SIZE = 16384;
private static ObjectOpenHashSet<GTRecipeInput> INSTANCES;
private static final String DATA_NAME = "expectedIngredientInstances";

private GTRecipeInputCache() {}

public static boolean isCacheEnabled() {
return INSTANCES != null;
return instances != null;
}

@ApiStatus.Internal
public static void enableCache() {
if (!isCacheEnabled()) {
INSTANCES = new ObjectOpenHashSet<>(EXPECTED_CACHE_SIZE, 1);
if (ConfigHolder.misc.debug || GTValues.isDeobfEnvironment())
GTLog.logger.info("GTRecipeInput cache enabled");
int size = calculateOptimalExpectedSize();
instances = new ObjectOpenHashSet<>(size);

if (ConfigHolder.misc.debug || GTValues.isDeobfEnvironment()) {
GTLog.logger.info("GTRecipeInput cache enabled with expected size {}", size);
}
}
}

@ApiStatus.Internal
public static void disableCache() {
if (isCacheEnabled()) {
if (ConfigHolder.misc.debug || GTValues.isDeobfEnvironment())
GTLog.logger.info("GTRecipeInput cache disabled; releasing {} unique instances", INSTANCES.size());
INSTANCES = null;
int size = instances.size();
if (ConfigHolder.misc.debug || GTValues.isDeobfEnvironment()) {
GTLog.logger.info("GTRecipeInput cache disabled; releasing {} unique instances", size);
}
instances = null;

if (size >= MINIMUM_CACHE_SIZE && size < MAXIMUM_CACHE_SIZE) {
NBTTagCompound tagCompound = PersistentData.instance().getTag();
if (getExpectedInstanceAmount(tagCompound) != size) {
tagCompound.setInteger(DATA_NAME, size);
PersistentData.instance().save();
}
}
}
}

private static int getExpectedInstanceAmount(@NotNull NBTTagCompound tagCompound) {
return MathHelper.clamp(tagCompound.getInteger(DATA_NAME), MINIMUM_CACHE_SIZE, MAXIMUM_CACHE_SIZE);
}

/**
* Tries to deduplicate the instance with previously cached instances.
* If there is no identical GTRecipeInput present in cache, the
* {@code recipeInput} will be put into cache, marked as cached, and returned subsequently.
* Tries to deduplicate the instance with previously cached instances. If there is no identical GTRecipeInput
* present in cache, the {@code recipeInput} will be put into cache, marked as cached, and returned subsequently.
* <p>
* This operation returns {@code recipeInput} without doing anything if cache is disabled.
*
* @param recipeInput ingredient instance to be deduplicated
* @return Either previously cached instance, or {@code recipeInput} marked cached;
* or unmodified {@code recipeInput} instance if the cache is disabled
* @return Either previously cached instance, or {@code recipeInput} marked cached; or unmodified
* {@code recipeInput} instance if the cache is disabled
*/
public static GTRecipeInput deduplicate(GTRecipeInput recipeInput) {
if (!isCacheEnabled() || recipeInput.isCached()) {
return recipeInput;
}
GTRecipeInput cached = INSTANCES.addOrGet(recipeInput);
GTRecipeInput cached = instances.addOrGet(recipeInput);
if (cached == recipeInput) { // If recipeInput is cached just now...
cached.setCached();
}
return cached;
}

/**
* Tries to deduplicate each instance in the list with previously cached instances.
* If there is no identical GTRecipeInput present in cache, the
* {@code recipeInput} will be put into cache, marked as cached, and returned subsequently.
* Tries to deduplicate each instance in the list with previously cached instances. If there is no identical
* GTRecipeInput present in cache, the {@code recipeInput} will be put into cache, marked as cached, and returned
* subsequently.
* <p>
* This operation returns {@code inputs} without doing anything if cache is disabled.
*
Expand All @@ -91,4 +121,50 @@ public static List<GTRecipeInput> deduplicateInputs(List<GTRecipeInput> inputs)
}
return list;
}

/**
* Calculates the optimal expected size for the input cache:
* <ol>
* <li>Pick a Load Factor to test: i.e. {@code 0.75f} (default).</li>
* <li>Pick a Size to test: i.e. {@code 8192}.</li>
* <li>Internal array's size: next highest power of 2 for {@code size / loadFactor},
* {@code nextHighestPowerOf2(8192 / 0.75) = 16384}.</li>
* <li>The maximum amount of stored values before a rehash is required {@code arraySize * loadFactor},
* {@code 16384 * 0.75 = 12288}.</li>
* <li>Compare with the known amount of values stored: {@code 12288 >= 11774}.</li>
* <li>If larger or equal, the initial capacity and load factor will not induce a rehash/resize.</li>
* </ol>
*
* @return the optimal expected input cache size
*/
private static int calculateOptimalExpectedSize() {
int min = Math.max(getExpectedInstanceAmount(PersistentData.instance().getTag()), MINIMUM_CACHE_SIZE);
for (int i = 13; i < 31; i++) {
int sizeToTest = 1 << i;
int arraySize = nextHighestPowerOf2((int) (sizeToTest / Hash.DEFAULT_LOAD_FACTOR));
int maxStoredBeforeRehash = (int) (arraySize * Hash.DEFAULT_LOAD_FACTOR);

if (maxStoredBeforeRehash >= min) {
return sizeToTest;
}
}
return MINIMUM_CACHE_SIZE;
}

/**
* <a href="https://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2">Algorithm source.</a>
*
* @param x the number to use
* @return the next highest power of 2 relative to the number
*/
private static int nextHighestPowerOf2(int x) {
x--;
x |= x >> 1;
x |= x >> 2;
x |= x >> 4;
x |= x >> 8;
x |= x >> 16;
x++;
return x;
}
}
Loading