diff --git a/README.md b/README.md index 169af43..a72fcac 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,45 @@ Then add PUUIDs dependency from Github: ``` +# MySQL +PUUIDs can mirror everything in its `Data` folder into MySQL (or MariaDB). It's off by default - turn it on under `MySQL` in your `config.yml`: + +```yaml +MySQL: + Enabled: true + Host: localhost + Port: 3306 + Database: minecraft + Username: root + Password: '' +``` + +The `.yml` files stay in charge. Every API read still comes off disk, so switching this on can't slow a plugin down or fail because the database is busy - each write is simply copied up in the background as well. That gives you one place to back up or query, and a way to share player data between the servers on a network. + +### How it's laid out +Every plugin that stores data gets its own table, so you can look at one plugin's rows on their own (and drop the table when you stop using it): + +| Table | What's in it | +| --- | --- | +| `puuids_players` | One row per player: username, IP, last seen, total play time. | +| `puuids_plugins` | Which table belongs to which plugin. | +| `puuids_data_` | One row per value that plugin has stored: `uuid`, `path`, `value`. | + +Values keep their type on the round trip - an int comes back an int, a list comes back a list, and ItemStacks come back as themselves. + +### Commands +| Command | What it does | +| --- | --- | +| `/puuids mysql` | Connection state, queue depth, and anything that has gone wrong. | +| `/puuids mysql export` | Pushes the whole `Data` folder up. Run this the first time you switch MySQL on. | +| `/puuids mysql import confirm` | Pulls everything down, overwriting local values with the database's. | +| `/puuids mysql reconnect` | Retries a connection that was down at start-up. | + +### For a network +Set `Sync-On-Join: true` and a player's file is refreshed from the shared database as they join, so their data follows them between servers. A file that has seen them more recently than the database wins, so nothing is lost when they hop back. On a brand new server, `Import-On-Startup: true` builds the folder from the database before anything reads it. + +If the database goes away, puuids keeps saving to file and queues the changes; they're sent as soon as it comes back. Your server needs a MySQL or MariaDB JDBC driver on its classpath - most Spigot and Paper builds ship one, and puuids says so in the console if yours doesn't. + # Spigot PUUIDs is a Spigot plugin for MC versions 1.13-1.21. Please check out the [Spigot Page](https://www.spigotmc.org/resources/puuids-•-an-async-file-api.71496/). for full documentation. diff --git a/src/com/zachduda/puuids/Cooldowns.java b/src/com/zachduda/puuids/Cooldowns.java index 8105ffd..9b9ebbd 100644 --- a/src/com/zachduda/puuids/Cooldowns.java +++ b/src/com/zachduda/puuids/Cooldowns.java @@ -57,6 +57,15 @@ static boolean onTimeCooling(UUID p) { return active(ontime, p); } + /** + * Drops a leaving player's /ontime cooldown. Without this the map keeps an entry for every + * player who ever ran the command and never came back. + */ + @SuppressWarnings("SpellCheckingInspection") + static void forgetOnTime(UUID p) { + ontime.remove(p); + } + @SuppressWarnings("SpellCheckingInspection") static void onTime(UUID p) { if (!plugin.getConfig().getBoolean("Settings.Cooldowns.On-Time.Enabled", true)) { diff --git a/src/com/zachduda/puuids/Main.java b/src/com/zachduda/puuids/Main.java index 2e1ed1a..f052722 100644 --- a/src/com/zachduda/puuids/Main.java +++ b/src/com/zachduda/puuids/Main.java @@ -3,6 +3,9 @@ import com.zachduda.puuids.api.PUUIDS.*; import com.zachduda.puuids.api.*; import com.zachduda.puuids.api.VersionManager.VersionTest; +import com.zachduda.puuids.storage.FileStore; +import com.zachduda.puuids.storage.MySQLSettings; +import com.zachduda.puuids.storage.MySQLStorage; import com.earth2me.essentials.Essentials; import com.earth2me.essentials.User; import com.google.common.io.Files; @@ -67,6 +70,9 @@ public class Main extends JavaPlugin implements Listener { // Lower-cased username -> UUID, so name lookups don't have to parse every file on disk. private final Map nameindex = new ConcurrentHashMap<>(); + // Optional MySQL mirror of the Data folder. Null whenever it is off or couldn't connect. + private volatile MySQLStorage storage; + private Metrics metrics; public void onEnable() { @@ -103,32 +109,149 @@ public void onEnable() { saveConfig(); updateConfig(); - final boolean useclean = getConfig().getBoolean("Settings.File-Cleanup.Enabled"); - final boolean cleaness = getConfig().getBoolean("Settings.File-Cleanup.Clean-Essentials"); + /* + When the database is the master copy - a fresh server joining a network, or one being + restored - the folder has to be rebuilt before anything reads or cleans it, so the scan + waits for the import rather than racing it. + */ + final MySQLStorage atstartup = storage; + if (atstartup != null && atstartup.settings().importonstartup) { + getLogger().info("Importing player data from MySQL before start-up checks..."); + atstartup.importAll(line -> Msgs.sendPrefix(Bukkit.getConsoleSender(), line), + () -> mpl.scheduling().asyncScheduler().run(this::startupScan)); + } else { + mpl.scheduling().asyncScheduler().run(this::startupScan); + } + + plugins.put(this, APIVersion.V4); + allowconnections = true; + + Bukkit.getServer().getPluginManager().registerEvents(this, this); + + // Idempotent; updateConfig() above already started it at the configured rate. The timer + // used to be a static final field built during class-load with the hardcoded defaults, + // so Advanced.Save-Rate-Ticks never had any effect at all. + Timer.startTimer(); + + mpl.scheduling().globalRegionalScheduler().run(() -> { + + if (debug) { + int total = plugins.size() - 1; + if (total == 1) { + debug("Hooked with " + total + " plugin."); + } else if (total == 0) { + debug("There aren't any plugins hooked with PUUIDS yet."); + } else { + debug("Hooked with " + total + " plugins."); + } + } + if(!getConfig().getBoolean("Advanced.Allow-Post-Startup-Connections")) { + allowconnections = false; + debug("Plugin registration window is now locked."); + } + + // Plugman -- Prevent Reloading + if (getServer().getPluginManager().isPluginEnabled("PlugMan")) { + debug("Detected PlugMan..."); + Plugin plugMan = Bukkit.getPluginManager().getPlugin("PlugMan"); + try { + List ignoredPlugins = (List) Objects.requireNonNull(plugMan).getClass().getMethod("getIgnoredPlugins").invoke(plugMan); + if (!ignoredPlugins.contains("PUUIDs")) { + ignoredPlugins.add("PUUIDs"); + debug("Injecting exception into Plugman..."); + } + } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException ignored) { + if(debug) { + debug("[Do Not Report] There was an issue when trying to communicate to PlugMan: "); + ignored.printStackTrace(); + } + } + } + // --- End of Plugman Prevention + }); + + if (updatecheck) { + new Updater(this, mpl).checkForUpdate(); + } + + // Players shouldn't EVER be online when we are starting.... + Collection players = Bukkit.getOnlinePlayers(); + if (!players.isEmpty()) { + status = false; + statusreason = "" + + " was improperly reloaded. This may damage your player's data files! Please restart your server."; + Msgs.sendPrefix(Bukkit.getConsoleSender(), "&4&l &c&l&nReloading puuids without a proper restart can severely damage PUUID's player data. PLEASE RESTART YOUR SERVER!"); + for (Player online : players) { + if (online.isOp() || online.hasPermission("puuids.admin")) { + Msgs.sendPrefix(online, "&c&lWARNING: &fPUUIDs has been improperly reloaded. This will cause data loss and possible damage to other plugins."); + if (sounds) { + online.playSound(online.getLocation(), Sound.ENTITY_ELDER_GUARDIAN_CURSE, 2.0F, 2.0F); + } + } + } + } + + if (getConfig().getBoolean("Settings.Metrics", true)) { + metrics = new Metrics(this, 18290); + } else { + debug("Metrics have been disabled in the config.yml. Guess we won't support all this hard work today!"); + } + + // Anyone already online (an improper reload) still needs a play time session opened, + // or nothing would accrue for them until they reconnected. + for (Player online : players) { + Timer.startSession(online.getUniqueId()); + } + + taskresettimer = startStatResetTimer(); + playerupdatetimer = startPlayerUpdateTimer(); mpl.scheduling().asyncScheduler().run(() -> { + ConnectionOpen coe = new ConnectionOpen(); + Bukkit.getPluginManager().callEvent(coe); + }); + } + + /** + * Walks the Data folder once at start-up: drops files that are corrupt or long abandoned, + * fills the username index, and reconciles play time with the server's own statistic. + *

+ * Runs off the main thread, and always ends by clearing {@code asyncrunning} - the file + * writer is held until it does. + */ + private void startupScan() { + try { + final boolean useclean = getConfig().getBoolean("Settings.File-Cleanup.Enabled", true); + final boolean cleaness = getConfig().getBoolean("Settings.File-Cleanup.Clean-Essentials", true); + // Without the default a missing (or mistyped) key read as 0 days, which deletes the + // entire Data folder on the next start-up. + final int maxDays = Math.max(1, getConfig().getInt("Settings.File-Cleanup.Max-Days", 365)); + final Essentials ess = (Essentials) Bukkit.getPluginManager().getPlugin("Essentials"); final File folder = new File(this.getDataFolder(), File.separator + "Data"); final File[] cachefiles = folder.exists() ? folder.listFiles() : null; if (cachefiles == null) { - asyncrunning = false; return; } ArrayList unknownfiles = new ArrayList<>(); - int maxDays = getConfig().getInt("Settings.File-Cleanup.Max-Days"); - for (File cachefile : cachefiles) { String path = cachefile.getPath(); final File f = new File(path); if (!Files.getFileExtension(path).equalsIgnoreCase("yml")) { - if(f.getName().toLowerCase().contains("ds_store")) { - if(!f.delete()) { + if (f.getName().toLowerCase().contains("ds_store")) { + if (!f.delete()) { debug("Error deleting file:" + f.toPath()); } debug("Found macOS .ds_store file in folder. Deleting!"); + } else if (f.getName().endsWith(FileStore.TEMP_SUFFIX)) { + // Left behind by a save that was interrupted; the real file is intact. + if (!f.delete()) { + debug("Error deleting file:" + f.toPath()); + } + debug("Cleaned up a leftover temporary file: " + f.getName()); } else { unknownfiles.add(f.getName()); } @@ -193,7 +316,7 @@ public void onEnable() { if(playtime > puuids_playtime) { debug("Using native MC playtime for puuids data file for " + playername); setcache.set("Time-Played", playtime); - setcache.save(f); + FileStore.save(setcache, f); } if (debug) { if (useclean) { @@ -226,100 +349,19 @@ public void onEnable() { statusreason = "Unknown file was found in your puuids Data folder, please remove the following files: " + unknownfiles; } - asyncrunning = false; unknownfiles.clear(); - }); // End of Async; - - plugins.put(this, APIVersion.V4); - allowconnections = true; - - Bukkit.getServer().getPluginManager().registerEvents(this, this); - - // Idempotent; updateConfig() above already started it at the configured rate. The timer - // used to be a static final field built during class-load with the hardcoded defaults, - // so Advanced.Save-Rate-Ticks never had any effect at all. - Timer.startTimer(); - - mpl.scheduling().globalRegionalScheduler().run(() -> { - - if (debug) { - int total = plugins.size() - 1; - if (total == 1) { - debug("Hooked with " + total + " plugin."); - } else if (total == 0) { - debug("There aren't any plugins hooked with PUUIDS yet."); - } else { - debug("Hooked with " + total + " plugins."); - } - } - if(!getConfig().getBoolean("Advanced.Allow-Post-Startup-Connections")) { - allowconnections = false; - debug("Plugin registration window is now locked."); - } - - // Plugman -- Prevent Reloading - if (getServer().getPluginManager().isPluginEnabled("PlugMan")) { - debug("Detected PlugMan..."); - Plugin plugMan = Bukkit.getPluginManager().getPlugin("PlugMan"); - try { - List ignoredPlugins = (List) Objects.requireNonNull(plugMan).getClass().getMethod("getIgnoredPlugins").invoke(plugMan); - if (!ignoredPlugins.contains("PUUIDs")) { - ignoredPlugins.add("PUUIDs"); - debug("Injecting exception into Plugman..."); - } - } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException ignored) { - if(debug) { - debug("[Do Not Report] There was an issue when trying to communicate to PlugMan: "); - ignored.printStackTrace(); - } - } - } - // --- End of Plugman Prevention - }); - - if (updatecheck) { - new Updater(this, mpl).checkForUpdate(); - } - - // Players shouldn't EVER be online when we are starting.... - Collection players = Bukkit.getOnlinePlayers(); - if (!players.isEmpty()) { - status = false; - statusreason = "" + - " was improperly reloaded. This may damage your player's data files! Please restart your server."; - Msgs.sendPrefix(Bukkit.getConsoleSender(), "&4&l &c&l&nReloading puuids without a proper restart can severely damage PUUID's player data. PLEASE RESTART YOUR SERVER!"); - for (Player online : players) { - if (online.isOp() || online.hasPermission("puuids.admin")) { - Msgs.sendPrefix(online, "&c&lWARNING: &fPUUIDs has been improperly reloaded. This will cause data loss and possible damage to other plugins."); - if (sounds) { - online.playSound(online.getLocation(), Sound.ENTITY_ELDER_GUARDIAN_CURSE, 2.0F, 2.0F); - } - } - } + } finally { + // Saving stays frozen until this returns, however it returns. + asyncrunning = false; } - if (getConfig().getBoolean("Settings.Metrics", true)) { - metrics = new Metrics(this, 18290); - } else { - debug("Metrics have been disabled in the config.yml. Guess we won't support all this hard work today!"); + final MySQLStorage started = storage; + if (started != null && started.settings().exportonstartup) { + getLogger().info("Exporting the Data folder to MySQL..."); + started.exportAll(line -> Msgs.sendPrefix(Bukkit.getConsoleSender(), line), null); } - - // Anyone already online (an improper reload) still needs a play time session opened, - // or nothing would accrue for them until they reconnected. - for (Player online : players) { - Timer.startSession(online.getUniqueId()); - } - - taskresettimer = startStatResetTimer(); - playerupdatetimer = startPlayerUpdateTimer(); - - mpl.scheduling().asyncScheduler().run(() -> { - ConnectionOpen coe = new ConnectionOpen(); - Bukkit.getPluginManager().callEvent(coe); - }); } - /* Checkpoints every online player's file so play time survives a crash. A normal quit is always exact; this interval only bounds how much of a session is lost if the server dies @@ -359,6 +401,55 @@ public void debug(String input) { } } + public boolean isDebug() { + return debug; + } + + /** The MySQL mirror, or null when it is switched off or couldn't connect. */ + public MySQLStorage getStorage() { + return storage; + } + + /** + * Whether the file writer is currently held. Queued changes accumulate while it is, which is + * how the start-up scan, the reset commands and a MySQL import all keep the folder to + * themselves. + */ + public boolean isSavingPaused() { + return asyncrunning; + } + + public void setSavingPaused(boolean paused) { + asyncrunning = paused; + } + + /** + * Starts, stops or rebuilds the MySQL mirror so it matches the config. Safe to call on a + * reload: an unchanged MySQL section leaves the existing connection alone. + */ + private void applyStorageConfig() { + final MySQLSettings updated = MySQLSettings.from(getConfig()); + final MySQLStorage current = storage; + + if (current != null && current.settings().sameAs(updated)) { + return; + } + + if (current != null) { + storage = null; + current.shutdown(); + } + + if (!updated.enabled) { + return; + } + + final MySQLStorage created = new MySQLStorage(this, updated); + if (created.start()) { + storage = created; + } + } + public void onDisable() { final long start = System.currentTimeMillis(); @@ -386,6 +477,14 @@ public void onDisable() { } Timer.stopTimer(); + + // After the file writer, so every last change it made is mirrored before we disconnect. + final MySQLStorage closing = storage; + storage = null; + if (closing != null) { + closing.shutdown(); + } + mpl.scheduling().cancelGlobalTasks(); plugins.clear(); @@ -410,9 +509,11 @@ private void updateConfig() { } debug("Configuration version is: " + conf_ver); - if((Objects.equals(getConfig().getString("Advanced.UUID"), "0")) || (conf_ver < 2)) { + // The id is read back from Advanced.UUID, but used to be written to a top-level "UUID" + // key - so getServerId() always answered "0" and a fresh id was generated every start-up. + if(Objects.equals(getConfig().getString("Advanced.UUID"), "0") || getConfig().getString("Advanced.UUID") == null) { debug("Generating new server UUID for saving..."); - getConfig().set("UUID", UUID.randomUUID().toString()); + getConfig().set("Advanced.UUID", UUID.randomUUID().toString()); } if(conf_ver < 2) { @@ -462,10 +563,14 @@ private void updateConfig() { debug("Sounds have been disabled, this is an older version of Minecraft."); } - if(getConfig().getBoolean("Advanced.Allow-Unsafe-Reloads")) { - allow_unsafe_reloads = true; + // Assigned either way: turning the option back off and reloading used to leave unsafe + // reloads permitted until the next restart. + allow_unsafe_reloads = getConfig().getBoolean("Advanced.Allow-Unsafe-Reloads", false); + if(allow_unsafe_reloads) { getLogger().warning("Unsafe reloading (via /rl, /reload, /restart) is permitted per config.yml. This is VERY dangerous! You will get no support for corrupted files."); } + + applyStorageConfig(); } @@ -564,7 +669,7 @@ public int set(Plugin pl, String uuid, Object should_be_null) { } /** Remembers a username so the next lookup for it doesn't have to scan the data folder. */ - void indexName(String name, String uuid) { + public void indexName(String name, String uuid) { if (name == null || uuid == null) { return; } @@ -748,6 +853,13 @@ public void onJoin(PlayerJoinEvent e) { // file refresh below is skipped by the cooldown, or a quick reconnect accrues nothing. Timer.startSession(joining.getUniqueId()); + // On a network the player may have been on another server since we last saw them, so + // their file is refreshed from the shared database before anything reads it. + final MySQLStorage db = storage; + if (db != null && db.isConnected() && db.settings().synconjoin) { + db.pullPlayer(joining.getUniqueId().toString()); + } + mpl.scheduling().asyncScheduler().run(() -> { Player p = joining; UUID uuid = p.getUniqueId(); @@ -795,6 +907,7 @@ public void onQuit(PlayerQuitEvent e) { updateFile(p, true); Cooldowns.justJoined(uuid); Cooldowns.clearConfirm(uuid); + Cooldowns.forgetOnTime(uuid); } private String randomString() { @@ -898,11 +1011,16 @@ public boolean onCommand(CommandSender sender, Command cmd, String cmdLabel, Str Msgs.send(sender, "&8&l> &f&l/puuids reset ontime &7Set everyone's total play-time back to 0."); } Msgs.send(sender, "&8&l> &f&l/puuids plugins &7Shows connected plugins."); + Msgs.send(sender, "&8&l> &f&l/puuids mysql &7Check on (or re-sync) the MySQL mirror."); Msgs.send(sender, ""); pop(sender); return true; } + if (args[0].equalsIgnoreCase("mysql")) { + return mysqlCommand(sender, args); + } + if (args[0].equalsIgnoreCase("plugins")) { if (plugins.isEmpty()) { pop(sender); @@ -1106,6 +1224,7 @@ public boolean onCommand(CommandSender sender, Command cmd, String cmdLabel, Str mpl.scheduling().asyncScheduler().run(() -> { int total = 0; final long start = System.currentTimeMillis(); + final MySQLStorage db = storage; for (File AllData : Objects.requireNonNull(folder.listFiles())) { File f = new File(AllData.getPath()); @@ -1114,8 +1233,15 @@ public boolean onCommand(CommandSender sender, Command cmd, String cmdLabel, Str setcache.set("Time-Played", 0); try { - setcache.save(f); + FileStore.save(setcache, f); + if (db != null && db.isConnected()) { + // Or the database would hand the old play time straight back + // on the next sync. + db.mirrorPlayer(setcache.getString("UUID"), setcache.getString("Username"), + setcache.getString("IP"), setcache.getLong("Last-On"), 0); + } } catch (Exception err) { + debug("Unable to reset " + f.getName() + ": " + err); } total++; } @@ -1186,6 +1312,7 @@ public boolean onCommand(CommandSender sender, Command cmd, String cmdLabel, Str mpl.scheduling().asyncScheduler().run(() -> { int total = 0; final long start = System.currentTimeMillis(); + final List wiped = new ArrayList<>(); for (File AllData : Objects.requireNonNull(folder.listFiles())) { File f = new File(AllData.getPath()); @@ -1195,12 +1322,21 @@ public boolean onCommand(CommandSender sender, Command cmd, String cmdLabel, Str debug("Reset" + setcache.getString("Username") + "'s file back to basics. (" + f.getName() + ")"); try { - setcache.save(f); + FileStore.save(setcache, f); + wiped.add(setcache.getString("UUID")); } catch (Exception err) { + debug("Unable to reset " + f.getName() + ": " + err); } total++; } + final MySQLStorage db = storage; + if (db != null && db.isConnected()) { + // The same rows have to go from MySQL, or a later import or join + // sync would put everything that was just erased back again. + db.clearPluginData(wiped, line -> Msgs.sendPrefix(sender, line)); + } + final String finished = Long.toString(System.currentTimeMillis() - start); getLogger().info("Reset " + total + " players files back to basics. (Done in " + finished + "ms)"); @@ -1358,20 +1494,102 @@ public boolean onCommand(CommandSender sender, Command cmd, String cmdLabel, Str return true; } + /* + Every connected plugin's name, comma separated. The old version appended the last plugin + twice and always left a trailing separator dangling on the end of the list. + */ private StringBuilder getStringBuilder() { StringBuilder sb = new StringBuilder(); - int plsb = 0; for(HashMap.Entry entry : plugins.entrySet()) { final String plname = entry.getKey().getDescription().getName(); if (!plname.equalsIgnoreCase("puuids")) { - if (plsb == getPlugins().size() - 1) { - sb.append(plname); + if (sb.length() > 0) { + sb.append("&f, &e"); } - - sb.append(plname).append("&f, &e"); - plsb++; + sb.append(plname); } } return sb; } + + /** + * {@code /puuids mysql [export|import confirm|reconnect]}. + *

+ * With no argument it reports on the mirror. The rest are the two directions of a manual + * re-sync plus a way to retry a connection that was down when the server started. + */ + private boolean mysqlCommand(CommandSender sender, String[] args) { + final MySQLStorage db = storage; + + if (args.length == 1) { + Msgs.send(sender, ""); + Msgs.send(sender, "#4ec483&lPUUIDs &8&l| &fMySQL"); + if (db == null) { + if (getConfig().getBoolean("MySQL.Enabled", false)) { + Msgs.send(sender, "&8&l> &c&lNot Connected. &fCheck the console, then &7/puuids mysql reconnect"); + } else { + Msgs.send(sender, "&8&l> &7&lDisabled. &fSet &fMySQL.Enabled &fto true in your config.yml."); + } + } else { + for (String line : db.status()) { + Msgs.send(sender, line); + } + Msgs.send(sender, "&8&l> &7&o/puuids mysql export &8- &7push the Data folder to MySQL"); + Msgs.send(sender, "&8&l> &7&o/puuids mysql import confirm &8- &7overwrite the Data folder from MySQL"); + } + Msgs.send(sender, ""); + pop(sender); + return true; + } + + if (args[1].equalsIgnoreCase("reconnect")) { + Msgs.sendPrefix(sender, "&7&oReconnecting to MySQL..."); + thinking(sender); + if (db != null) { + storage = null; + db.shutdown(); + } + applyStorageConfig(); + if (storage == null) { + bass(sender); + Msgs.sendPrefix(sender, "&c&lStill Down. &fSee the console for what MySQL said."); + } else { + pop(sender); + Msgs.sendPrefix(sender, "&a&lConnected. &fPlayer data is being mirrored again."); + } + return true; + } + + if (db == null || !db.isConnected()) { + bass(sender); + Msgs.sendPrefix(sender, "&c&lNot Connected. &fMySQL has to be on and reachable for that."); + return true; + } + + if (args[1].equalsIgnoreCase("export")) { + Msgs.sendPrefix(sender, "&7&oExporting to MySQL, this may take a while..."); + thinking(sender); + db.exportAll(line -> Msgs.sendPrefix(sender, line), null); + return true; + } + + if (args[1].equalsIgnoreCase("import")) { + // This overwrites files with whatever the database holds, so it is never one word. + if (args.length < 3 || !args[2].equalsIgnoreCase("confirm")) { + bass(sender); + Msgs.sendPrefix(sender, "&c&lARE YOU SURE? &fEvery value in MySQL will overwrite the one in your Data folder."); + Msgs.sendPrefix(sender, "&fRun &7&l/puuids mysql import confirm&f if that is what you want."); + return true; + } + + Msgs.sendPrefix(sender, "&7&oImporting from MySQL, saving is paused until it finishes..."); + thinking(sender); + db.importAll(line -> Msgs.sendPrefix(sender, line), null); + return true; + } + + bass(sender); + Msgs.sendPrefix(sender, "&c&lOops. &fTry &7/puuids mysql &f(export/import/reconnect)"); + return true; + } } \ No newline at end of file diff --git a/src/com/zachduda/puuids/Timer.java b/src/com/zachduda/puuids/Timer.java index 8561321..7ee4186 100644 --- a/src/com/zachduda/puuids/Timer.java +++ b/src/com/zachduda/puuids/Timer.java @@ -2,6 +2,8 @@ import com.zachduda.puuids.api.OnNewFile; import com.zachduda.puuids.api.TimerSaved; +import com.zachduda.puuids.storage.FileStore; +import com.zachduda.puuids.storage.MySQLStorage; import org.bukkit.Bukkit; import org.bukkit.configuration.file.FileConfiguration; import org.bukkit.configuration.file.YamlConfiguration; @@ -10,8 +12,6 @@ import java.io.File; import java.net.InetSocketAddress; -import java.nio.file.Files; -import java.nio.file.StandardCopyOption; import java.time.Duration; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -144,6 +144,14 @@ private static void writeBatch(String uuid, Batch batch, boolean events) { final FileConfiguration setcache = YamlConfiguration.loadConfiguration(f); + /* + A file created by plugin data alone (a set for someone who has never joined) used to be + written without a UUID key, which the start-up scan then treats as corrupt and deletes. + */ + if (!setcache.contains("UUID")) { + setcache.set("UUID", uuid); + } + for (PlayerUpdate update : batch.updates) { setcache.set("UUID", update.uuid); setcache.set("Username", update.name); @@ -178,6 +186,8 @@ private static void writeBatch(String uuid, Batch batch, boolean events) { return; } + mirror(uuid, batch, setcache, isnewfile); + if (!batch.updates.isEmpty()) { plugin.setTimes.incrementAndGet(); plugin.indexName(batch.updates.get(batch.updates.size() - 1).name, uuid); @@ -211,27 +221,57 @@ private static void writeBatch(String uuid, Batch batch, boolean events) { * leave a player with a half-written (and therefore unreadable) data file. */ private static boolean save(FileConfiguration config, File target) { - final File temp = new File(target.getParentFile(), target.getName() + ".tmp"); try { - config.save(temp); - try { - Files.move(temp.toPath(), target.toPath(), - StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); - } catch (Exception atomicUnsupported) { - Files.move(temp.toPath(), target.toPath(), StandardCopyOption.REPLACE_EXISTING); - } + FileStore.save(config, target); return true; } catch (Exception err) { plugin.getLogger().warning("Unable to save puuids file " + target.getName() + ": " + err); if (plugin.debug) { err.printStackTrace(); } - //noinspection ResultOfMethodCallIgnored - temp.delete(); return false; } } + /** + * Hands the same changes to MySQL, if it is switched on. + *

+ * This runs after the file has been written, never instead of it: the file is still the + * source of truth, and a database that is slow or down only ever delays the copy. Values are + * read back out of the freshly saved config rather than taken from the queue, so what lands + * in MySQL is exactly what landed on disk. + */ + private static void mirror(String uuid, Batch batch, FileConfiguration setcache, boolean isnewfile) { + final MySQLStorage storage = plugin.getStorage(); + if (storage == null || !storage.isConnected()) { + return; + } + + try { + if (!batch.updates.isEmpty() || isnewfile) { + storage.mirrorPlayer(uuid, setcache.getString("Username"), setcache.getString("IP"), + setcache.getLong("Last-On"), setcache.getLong("Time-Played")); + } + + for (Quartet data : batch.sets) { + final String plname = data.getPlugin(); + final String path = data.getPath(); + + if (path.equals("PUUIDS_SET_AS_ALL_NULL")) { + storage.mirrorClear(plname, uuid); + } else { + storage.mirrorSet(plname, uuid, path, setcache.get("Plugins." + plname + "." + path)); + } + } + } catch (Exception err) { + // Mirroring is never allowed to break the file pipeline. + plugin.getLogger().warning("Unable to queue a MySQL update for " + uuid + ": " + err); + if (plugin.debug) { + err.printStackTrace(); + } + } + } + /** * Opens a play time session. Called the moment a player joins, before any of the * asynchronous join handling, so a session is never missed. diff --git a/src/com/zachduda/puuids/storage/ConnectionPool.java b/src/com/zachduda/puuids/storage/ConnectionPool.java new file mode 100644 index 0000000..7972aed --- /dev/null +++ b/src/com/zachduda/puuids/storage/ConnectionPool.java @@ -0,0 +1,117 @@ +package com.zachduda.puuids.storage; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * A deliberately small JDBC pool. + *

+ * Nothing here needs a full pooling library: writes come from one dedicated thread and the only + * other borrowers are the occasional import / export / join lookup. What it does have to get + * right is handing back a connection that is actually alive - MySQL closes idle connections + * after {@code wait_timeout} (eight hours by default), and a Minecraft server is idle at 4am. + */ +final class ConnectionPool { + + private final String url; + private final Properties properties; + private final int borrowtimeout; + + private final Semaphore permits; + private final ConcurrentLinkedQueue idle = new ConcurrentLinkedQueue<>(); + private final AtomicBoolean closed = new AtomicBoolean(false); + + ConnectionPool(String url, Properties properties, int size, int borrowtimeout) { + this.url = url; + this.properties = properties; + this.permits = new Semaphore(size, true); + this.borrowtimeout = borrowtimeout; + } + + /** + * Takes a live connection out of the pool, opening one if the pool isn't full yet. + * Every successful borrow must be matched by exactly one {@link #release(Connection)} or + * {@link #discard(Connection)}, or the pool leaks a permit and eventually deadlocks. + */ + Connection borrow() throws SQLException { + if (closed.get()) { + throw new SQLException("The puuids MySQL pool has been shut down."); + } + + try { + if (!permits.tryAcquire(borrowtimeout, TimeUnit.SECONDS)) { + throw new SQLException("Timed out waiting " + borrowtimeout + "s for a free MySQL connection."); + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new SQLException("Interrupted while waiting for a MySQL connection.", interrupted); + } + + try { + Connection pooled; + while ((pooled = idle.poll()) != null) { + if (usable(pooled)) { + return pooled; + } + closeQuietly(pooled); + } + return DriverManager.getConnection(url, properties); + } catch (SQLException | RuntimeException err) { + permits.release(); + throw err; + } + } + + /** Returns a healthy connection to the pool. */ + void release(Connection connection) { + if (connection == null) { + permits.release(); + return; + } + + if (closed.get()) { + closeQuietly(connection); + } else { + idle.add(connection); + } + permits.release(); + } + + /** Throws a connection away - use this after any error, the connection may be poisoned. */ + void discard(Connection connection) { + closeQuietly(connection); + permits.release(); + } + + void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + Connection pooled; + while ((pooled = idle.poll()) != null) { + closeQuietly(pooled); + } + } + + private static boolean usable(Connection connection) { + try { + return !connection.isClosed() && connection.isValid(2); + } catch (SQLException err) { + return false; + } + } + + private static void closeQuietly(Connection connection) { + try { + connection.close(); + } catch (SQLException ignored) { + // Already broken; there is nothing useful to do about it. + } + } +} diff --git a/src/com/zachduda/puuids/storage/FileStore.java b/src/com/zachduda/puuids/storage/FileStore.java new file mode 100644 index 0000000..e78f7d1 --- /dev/null +++ b/src/com/zachduda/puuids/storage/FileStore.java @@ -0,0 +1,43 @@ +package com.zachduda.puuids.storage; + +import org.bukkit.configuration.file.FileConfiguration; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; + +/** + * Writes a player file the safe way: to a sibling temp file first, then moved into place, so a + * crash halfway through a save can't leave somebody with a half-written (and therefore + * unreadable) data file. + */ +public final class FileStore { + + public static final String TEMP_SUFFIX = ".tmp"; + + private FileStore() { + } + + public static void save(FileConfiguration config, File target) throws IOException { + final File parent = target.getParentFile(); + if (parent != null && !parent.exists() && !parent.mkdirs() && !parent.isDirectory()) { + throw new IOException("Unable to create the data folder: " + parent); + } + + final File temp = new File(parent, target.getName() + TEMP_SUFFIX); + try { + config.save(temp); + try { + Files.move(temp.toPath(), target.toPath(), + StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException | UnsupportedOperationException atomicUnsupported) { + Files.move(temp.toPath(), target.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + } catch (IOException | RuntimeException err) { + //noinspection ResultOfMethodCallIgnored + temp.delete(); + throw err; + } + } +} diff --git a/src/com/zachduda/puuids/storage/MySQLSettings.java b/src/com/zachduda/puuids/storage/MySQLSettings.java new file mode 100644 index 0000000..6210648 --- /dev/null +++ b/src/com/zachduda/puuids/storage/MySQLSettings.java @@ -0,0 +1,120 @@ +package com.zachduda.puuids.storage; + +import org.bukkit.configuration.file.FileConfiguration; + +import java.util.Locale; +import java.util.Objects; + +/** + * An immutable snapshot of the MySQL section of config.yml. + *

+ * Taken once when the storage starts so a running flush never sees settings change underneath + * it, and compared on {@code /puuids reload} to decide whether the connection has to be rebuilt. + */ +public final class MySQLSettings { + + public final boolean enabled; + public final String host; + public final int port; + public final String database; + public final String username; + public final String password; + public final String prefix; + public final boolean usessl; + public final String extraproperties; + public final int poolsize; + public final int connecttimeout; + public final long flushms; + public final int maxqueued; + public final boolean importonstartup; + public final boolean exportonstartup; + public final boolean synconjoin; + + private MySQLSettings(FileConfiguration cfg) { + enabled = cfg.getBoolean("MySQL.Enabled", false); + host = text(cfg.getString("MySQL.Host", "localhost"), "localhost"); + port = clamp(cfg.getInt("MySQL.Port", 3306), 1, 65535, 3306); + database = text(cfg.getString("MySQL.Database", "minecraft"), "minecraft"); + username = text(cfg.getString("MySQL.Username", "root"), "root"); + // An empty password is legitimate, so this one only defends against a null. + password = cfg.getString("MySQL.Password", "") == null ? "" : cfg.getString("MySQL.Password", ""); + prefix = sanitizePrefix(cfg.getString("MySQL.Table-Prefix", "puuids_")); + usessl = cfg.getBoolean("MySQL.Use-SSL", false); + extraproperties = cfg.getString("MySQL.Extra-Properties", "") == null + ? "" : cfg.getString("MySQL.Extra-Properties", ""); + poolsize = clamp(cfg.getInt("MySQL.Pool-Size", 3), 1, 16, 3); + connecttimeout = clamp(cfg.getInt("MySQL.Connection-Timeout-Seconds", 10), 1, 120, 10); + // Below ~100ms the writer thread spends more time waking up than working. + flushms = clamp(cfg.getInt("MySQL.Flush-Rate-Ms", 1000), 100, 600000, 1000); + maxqueued = clamp(cfg.getInt("MySQL.Max-Queued-Writes", 100000), 1000, 5000000, 100000); + importonstartup = cfg.getBoolean("MySQL.Import-On-Startup", false); + exportonstartup = cfg.getBoolean("MySQL.Export-On-Startup", false); + synconjoin = cfg.getBoolean("MySQL.Sync-On-Join", false); + } + + public static MySQLSettings from(FileConfiguration cfg) { + return new MySQLSettings(cfg); + } + + /** A blank entry in the config means "I didn't set this", not "connect with nothing". */ + private static String text(String value, String fallback) { + return value == null || value.trim().isEmpty() ? fallback : value.trim(); + } + + private static int clamp(int value, int min, int max, int fallback) { + if (value < min || value > max) { + return fallback; + } + return value; + } + + /** + * Table names are built from this, so anything that isn't a plain identifier character is + * dropped rather than escaped - the prefix ends up inside a {@code CREATE TABLE}. + */ + private static String sanitizePrefix(String raw) { + if (raw == null) { + return "puuids_"; + } + final String cleaned = raw.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9_]", ""); + if (cleaned.isEmpty()) { + return "puuids_"; + } + // Capped so there is always room for a plugin name inside MySQL's 64 character limit. + return cleaned.length() > 24 ? cleaned.substring(0, 24) : cleaned; + } + + /** + * Whether two snapshots point at the same server with the same credentials. A reload that + * only changes, say, the flush rate doesn't need to tear the connection down. + */ + public boolean sameConnection(MySQLSettings other) { + return other != null + && enabled == other.enabled + && port == other.port + && usessl == other.usessl + && poolsize == other.poolsize + && connecttimeout == other.connecttimeout + && Objects.equals(host, other.host) + && Objects.equals(database, other.database) + && Objects.equals(username, other.username) + && Objects.equals(password, other.password) + && Objects.equals(prefix, other.prefix) + && Objects.equals(extraproperties, other.extraproperties); + } + + /** True when nothing at all changed, so a reload can leave a working connection alone. */ + public boolean sameAs(MySQLSettings other) { + return sameConnection(other) + && flushms == other.flushms + && maxqueued == other.maxqueued + && importonstartup == other.importonstartup + && exportonstartup == other.exportonstartup + && synconjoin == other.synconjoin; + } + + /** Host and database only - never the credentials, this ends up in the server log. */ + public String describe() { + return host + ":" + port + "/" + database; + } +} diff --git a/src/com/zachduda/puuids/storage/MySQLStorage.java b/src/com/zachduda/puuids/storage/MySQLStorage.java new file mode 100644 index 0000000..89e7f28 --- /dev/null +++ b/src/com/zachduda/puuids/storage/MySQLStorage.java @@ -0,0 +1,1053 @@ +package com.zachduda.puuids.storage; + +import com.zachduda.puuids.Main; +import org.bukkit.configuration.ConfigurationSection; +import org.bukkit.configuration.file.FileConfiguration; +import org.bukkit.configuration.file.YamlConfiguration; + +import java.io.File; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; + +/** + * Mirrors the Data folder into MySQL. + *

+ * The .yml files stay the source of truth - every read the API serves still comes off disk, so + * turning this on can never make a get slower or fail because a database is down. What it adds + * is a copy of each write, queued and sent from a single background thread, so the same data is + * available to a web panel, a backup job, or the other servers on a network. + *

+ * Layout is one table per plugin ({@code puuids_data_}), plus {@code puuids_players} for + * puuids' own record of each player and {@code puuids_plugins} mapping plugin names to tables. + */ +public final class MySQLStorage { + + private static final String[] MYSQL_DRIVERS = {"com.mysql.cj.jdbc.Driver", "com.mysql.jdbc.Driver"}; + private static final String MARIADB_DRIVER = "org.mariadb.jdbc.Driver"; + + /** Players handled per round trip when importing, so a big database can't eat the heap. */ + private static final int IMPORT_PAGE = 250; + /** Files read per transaction when exporting. */ + private static final int EXPORT_BATCH = 200; + /** How long a repeated failure stays quiet in the console. */ + private static final long LOG_MUTE_MS = 60_000L; + + private final Main plugin; + private final MySQLSettings settings; + private final String playerstable; + private final String registrytable; + + private final LinkedBlockingDeque queue = new LinkedBlockingDeque<>(); + private final Set ensured = ConcurrentHashMap.newKeySet(); + private final Set longpathwarned = ConcurrentHashMap.newKeySet(); + + private final AtomicLong written = new AtomicLong(); + private final AtomicLong dropped = new AtomicLong(); + private final AtomicBoolean running = new AtomicBoolean(); + private final AtomicBoolean longtask = new AtomicBoolean(); + + private volatile ConnectionPool pool; + private volatile ScheduledExecutorService worker; + private volatile boolean connected; + private volatile String lasterror; + private volatile long quietuntil; + private volatile long retryafter; + + public MySQLStorage(Main plugin, MySQLSettings settings) { + this.plugin = plugin; + this.settings = settings; + this.playerstable = Tables.players(settings.prefix); + this.registrytable = Tables.registry(settings.prefix); + } + + // Lifecycle --------------------------------------------------------------------------- + + /** + * Connects, creates the two fixed tables, and starts the writer thread. + * + * @return false if MySQL couldn't be reached; puuids carries on with files only. + */ + public boolean start() { + if (!running.compareAndSet(false, true)) { + return connected; + } + + final String driver = resolveDriver(); + if (driver == null) { + plugin.getLogger().severe("MySQL is enabled in your config.yml, but this server has no MySQL JDBC driver."); + plugin.getLogger().severe("Add a MySQL (or MariaDB) connector to your server, or set MySQL.Enabled to false."); + running.set(false); + return false; + } + + plugin.debug("Using JDBC driver " + driver); + pool = new ConnectionPool(url(driver), properties(), settings.poolsize, settings.connecttimeout); + + try { + withConnection(connection -> { + try (Statement st = connection.createStatement()) { + st.executeUpdate(Tables.createPlayers(playerstable)); + st.executeUpdate(Tables.createRegistry(registrytable)); + } + return null; + }); + } catch (SQLException err) { + plugin.getLogger().severe("Unable to connect to MySQL at " + settings.describe() + ": " + err.getMessage()); + plugin.getLogger().severe("puuids will keep saving to file only. Fix the connection and run /puuids mysql reconnect."); + lasterror = err.getMessage(); + pool.close(); + pool = null; + running.set(false); + return false; + } + + connected = true; + lasterror = null; + + final ThreadFactory factory = task -> { + final Thread thread = new Thread(task, "PUUIDs-MySQL"); + thread.setDaemon(true); + return thread; + }; + final ScheduledExecutorService service = Executors.newSingleThreadScheduledExecutor(factory); + service.scheduleWithFixedDelay(this::flushQuietly, settings.flushms, settings.flushms, TimeUnit.MILLISECONDS); + worker = service; + + plugin.getLogger().info("Connected to MySQL at " + settings.describe() + ". Player data will be mirrored there."); + return true; + } + + /** Flushes whatever is still queued and closes the connections. Called from onDisable. */ + public void shutdown() { + final ScheduledExecutorService service = worker; + worker = null; + + if (service != null) { + service.shutdown(); + try { + if (!service.awaitTermination(5, TimeUnit.SECONDS)) { + service.shutdownNow(); + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + service.shutdownNow(); + } + } + + // Anything still queued is written here, on the shutdown thread, before we let go. + final int leftover = queue.size(); + if (leftover > 0 && connected) { + plugin.getLogger().info("Sending " + leftover + " leftover changes to MySQL..."); + } + + retryafter = 0; // A backoff from earlier must not skip the last flush. + for (int attempt = 0; attempt < 200 && !queue.isEmpty(); attempt++) { + final int before = queue.size(); + flushQuietly(); + if (queue.size() >= before) { + // No progress: the database is unreachable, and waiting won't change that. + break; + } + } + + final ConnectionPool open = pool; + pool = null; + if (open != null) { + open.close(); + } + + connected = false; + running.set(false); + queue.clear(); + ensured.clear(); + } + + public MySQLSettings settings() { + return settings; + } + + public boolean isConnected() { + return connected; + } + + public int queued() { + return queue.size(); + } + + /** Human readable lines for {@code /puuids mysql}. */ + public List status() { + final List lines = new ArrayList<>(); + lines.add("&8&l> &fServer: &e" + settings.describe()); + lines.add("&8&l> &fConnected: " + (connected ? "&a&lYES" : "&c&lNO")); + lines.add("&8&l> &fQueued Changes: &e" + queue.size()); + lines.add("&8&l> &fChanges Written: &e" + written.get()); + lines.add("&8&l> &fPlugin Tables: &e" + ensured.size()); + if (dropped.get() > 0) { + lines.add("&8&l> &fDropped Changes: &c" + dropped.get()); + } + if (lasterror != null) { + lines.add("&8&l> &6&lLAST ERROR: &f" + lasterror); + } + return lines; + } + + // Mirroring --------------------------------------------------------------------------- + + /** Mirrors puuids' own record of a player, taken from their file after it was written. */ + public void mirrorPlayer(String uuid, String username, String ip, long laston, long timeplayed) { + if (uuid == null) { + return; + } + enqueue(new Op.Player(playerstable, uuid, username, ip, laston, timeplayed)); + } + + /** + * Mirrors one value a plugin has just written. + * + * @param current the value as it now stands in the player's file - null means it was cleared. + */ + public void mirrorSet(String pluginname, String uuid, String path, Object current) { + if (uuid == null || pluginname == null || path == null) { + return; + } + + if (tooLong(pluginname, path, "mirroring")) { + return; + } + + final String table = Tables.data(settings.prefix, pluginname); + + if (current == null) { + // Setting null on file drops the value and everything under it; so does this. + enqueue(new Op.Remove(table, pluginname, uuid, path, true)); + return; + } + + // Anything that used to be nested under this path is gone now that it holds a value + // again, and would otherwise be read back as a phantom child on the next import. + enqueue(new Op.Remove(table, pluginname, uuid, path, false)); + + if (current instanceof ConfigurationSection) { + final ConfigurationSection section = (ConfigurationSection) current; + for (String key : section.getKeys(true)) { + final Object leaf = section.get(key); + if (leaf == null || leaf instanceof ConfigurationSection) { + continue; // Only the leaves carry data; the sections are implied by the paths. + } + queueValue(table, pluginname, uuid, path + "." + key, leaf); + } + return; + } + + queueValue(table, pluginname, uuid, path, current); + } + + /** Mirrors {@code setNull(plugin, uuid)} - everything that plugin stored for that player. */ + public void mirrorClear(String pluginname, String uuid) { + if (uuid == null || pluginname == null) { + return; + } + enqueue(new Op.Clear(Tables.data(settings.prefix, pluginname), pluginname, uuid)); + } + + private void queueValue(String table, String pluginname, String uuid, String path, Object value) { + if (tooLong(pluginname, path, "mirroring")) { + return; + } + enqueue(new Op.Set(table, pluginname, uuid, path, ValueCodec.encode(value))); + } + + /** + * Warns once about a path that can't be indexed, and only for the first few hundred of them: + * a plugin generating unbounded path names must not be able to grow this set for ever. + */ + private boolean tooLong(String pluginname, String path, String what) { + if (path.length() <= Op.MAX_PATH) { + return false; + } + if (longpathwarned.size() < 500 && longpathwarned.add(pluginname + "." + path)) { + plugin.getLogger().warning("Skipping " + pluginname + "'s path '" + path + "' when " + what + + " to MySQL: paths over " + Op.MAX_PATH + " characters can't be indexed."); + } + return true; + } + + private void enqueue(Op op) { + if (!connected) { + return; + } + + queue.add(op); + + /* + If MySQL has been unreachable for a while the queue must not be allowed to grow until + the server runs out of memory. The files are still correct, so the oldest changes are + the ones to lose - /puuids mysql export rebuilds the database from them afterwards. + */ + while (queue.size() > settings.maxqueued) { + if (queue.poll() == null) { + break; + } + final long total = dropped.incrementAndGet(); + if (total == 1 || quiet()) { + plugin.getLogger().warning("MySQL is too far behind; dropping queued changes (" + + total + " so far). Run /puuids mysql export once it is healthy again."); + } + } + } + + // Writing ----------------------------------------------------------------------------- + + private void flushQuietly() { + try { + flush(); + } catch (Throwable err) { + // The writer thread must survive anything, or mirroring silently stops for good. + fail("Unexpected error while writing to MySQL", err); + } + } + + private void flush() { + if (!connected || queue.isEmpty() || pool == null) { + return; + } + + if (System.currentTimeMillis() < retryafter) { + return; // Backing off after a failure. + } + + final List batch = new ArrayList<>(Math.min(queue.size(), 5000)); + Op op; + while (batch.size() < 5000 && (op = queue.poll()) != null) { + batch.add(op); + } + + if (batch.isEmpty()) { + return; + } + + final List ready = reduce(batch); + + try { + withConnection(connection -> { + ensureTables(connection, ready); + apply(connection, ready); + return null; + }); + written.addAndGet(ready.size()); + retryafter = 0; + lasterror = null; + } catch (SQLException err) { + /* + Put the work back at the front so ordering survives the retry, then back off - a + database that is down stays down for a while, and hammering it every flush just + fills the log. + */ + for (int i = ready.size() - 1; i >= 0; i--) { + queue.addFirst(ready.get(i)); + } + retryafter = System.currentTimeMillis() + 15_000L; + fail("Unable to write " + ready.size() + " changes to MySQL", err); + } + } + + /** Creates any table this batch touches that we haven't already seen this session. */ + private void ensureTables(Connection connection, List batch) throws SQLException { + // DDL commits implicitly in MySQL, so it has to happen before the transaction opens. + final Map missing = new LinkedHashMap<>(); + for (Op op : batch) { + if (op.plugin != null && !ensured.contains(op.table)) { + missing.put(op.table, op.plugin); + } + } + + if (missing.isEmpty()) { + return; + } + + for (Map.Entry entry : missing.entrySet()) { + try (Statement st = connection.createStatement()) { + st.executeUpdate(Tables.createData(entry.getKey())); + } + try (PreparedStatement ps = connection.prepareStatement("INSERT INTO `" + registrytable + + "` (`plugin`,`table_name`,`updated`) VALUES (?,?,?)" + + " ON DUPLICATE KEY UPDATE `table_name`=VALUES(`table_name`),`updated`=VALUES(`updated`)")) { + ps.setString(1, entry.getValue()); + ps.setString(2, entry.getKey()); + ps.setLong(3, System.currentTimeMillis()); + ps.executeUpdate(); + } + ensured.add(entry.getKey()); + plugin.debug("MySQL table " + entry.getKey() + " is ready for " + entry.getValue() + "."); + } + } + + /** + * Collapses a batch down to the state it ends in, and orders what is left so that + * operations sharing a statement sit together. + *

+ * A player who earns points ten times in a flush window is one row, not ten writes, and a + * value that was cleared and re-set is a single insert. What comes out is, per plugin + * table: the clears, then the removes, then the sets - which is safe because anything a + * later remove would have wiped has already been dropped here, and anything an earlier + * remove has to precede is a set that stays behind it. + */ + private List reduce(List ops) { + final Map players = new LinkedHashMap<>(); + final Map> bytable = new LinkedHashMap<>(); + final Map pending = new LinkedHashMap<>(); + + for (Op op : ops) { + if (op instanceof Op.Player) { + // Every player row is a whole-row upsert, so only the last one matters. + players.put(op.uuid, (Op.Player) op); + continue; + } + + final Pending forplayer = pending.computeIfAbsent(op.table + '' + op.uuid, key -> { + final Pending created = new Pending(); + bytable.computeIfAbsent(op.table, k -> new ArrayList<>()).add(created); + return created; + }); + forplayer.add(op); + } + + final List out = new ArrayList<>(ops.size()); + out.addAll(players.values()); + + for (List table : bytable.values()) { + for (Pending entry : table) { + if (entry.clear != null) { + out.add(entry.clear); + } + } + for (Pending entry : table) { + for (Op.Remove remove : entry.removes.values()) { + if (remove.includeself) { + out.add(remove); + } + } + } + for (Pending entry : table) { + for (Op.Remove remove : entry.removes.values()) { + if (!remove.includeself) { + out.add(remove); + } + } + } + for (Pending entry : table) { + out.addAll(entry.sets.values()); + } + } + + return out; + } + + /** What is still outstanding for one plugin table and one player. */ + private static final class Pending { + private Op.Clear clear; + private final Map removes = new LinkedHashMap<>(); + private final Map sets = new LinkedHashMap<>(); + + private void add(Op op) { + if (op instanceof Op.Clear) { + // Wipes the slate: nothing queued before it can still matter. + clear = (Op.Clear) op; + removes.clear(); + sets.clear(); + return; + } + + if (op instanceof Op.Remove) { + final Op.Remove remove = (Op.Remove) op; + sets.keySet().removeIf(remove::covers); + + if (clear != null) { + return; // The clear already takes these rows out. + } + + removes.keySet().removeIf(path -> path.startsWith(remove.path + ".")); + removes.put(remove.path, remove); + return; + } + + final Op.Set set = (Op.Set) op; + sets.put(set.path, set); + } + } + + /** + * Sends the batch in order, grouping neighbouring operations that share a statement into one + * JDBC batch. Order matters: a remove followed by a set has to reach MySQL that way round. + */ + private void apply(Connection connection, List batch) throws SQLException { + final boolean autocommit = connection.getAutoCommit(); + connection.setAutoCommit(false); + try { + int index = 0; + while (index < batch.size()) { + final Op first = batch.get(index); + final String key = first.batchKey(); + + try (PreparedStatement ps = connection.prepareStatement(first.sql())) { + while (index < batch.size() && batch.get(index).batchKey().equals(key)) { + batch.get(index).bind(ps); + ps.addBatch(); + index++; + } + ps.executeBatch(); + } + } + connection.commit(); + } catch (SQLException | RuntimeException err) { + try { + connection.rollback(); + } catch (SQLException ignored) { + // The connection is discarded by the caller either way. + } + throw err; + } finally { + try { + connection.setAutoCommit(autocommit); + } catch (SQLException ignored) { + // Same: this connection is on its way out if we got here through an error. + } + } + } + + // Export ------------------------------------------------------------------------------ + + /** + * Pushes every file in the Data folder up to MySQL. This is what you run once after turning + * the option on, and after any spell where the database was unreachable. + */ + public void exportAll(Consumer progress, Runnable then) { + final boolean queued = submit(() -> { + if (!longtask.compareAndSet(false, true)) { + progress.accept("&c&lBusy. &fAnother MySQL import or export is already running."); + finish(then); + return; + } + + final long start = System.currentTimeMillis(); + try { + final File[] files = dataFiles(); + if (files.length == 0) { + progress.accept("&6&lNothing To Do. &fThere are no data files to export."); + return; + } + + progress.accept("&7&oExporting " + files.length + " players to MySQL..."); + + int exported = 0; + final List batch = new ArrayList<>(); + + for (File file : files) { + final String uuid = file.getName().substring(0, file.getName().length() - 4); + final FileConfiguration data = YamlConfiguration.loadConfiguration(file); + collect(uuid, data, batch); + exported++; + + if (exported % EXPORT_BATCH == 0) { + write(batch); + batch.clear(); + progress.accept("&7&oExported " + exported + " / " + files.length + " players..."); + } + } + + write(batch); + progress.accept("&a&lDone. &fExported &7&l" + exported + "&f players to MySQL in &7&l" + + (System.currentTimeMillis() - start) + "ms"); + } catch (SQLException err) { + fail("Export to MySQL failed", err); + progress.accept("&c&lFailed. &fMySQL rejected the export: " + err.getMessage()); + } finally { + longtask.set(false); + finish(then); + } + }); + + if (!queued) { + // Nothing will ever run, so whoever is waiting on us has to be released here. + finish(then); + } + } + + /** + * Drops every plugin's rows for the given players, leaving their player record alone. + *

+ * This is what {@code /puuids reset all} does to the files, and it has to happen here too: + * otherwise a later import - or a join refresh - would hand back exactly the data an admin + * had just erased. + */ + public void clearPluginData(List uuids, Consumer progress) { + if (uuids == null || uuids.isEmpty()) { + return; + } + + final List targets = new ArrayList<>(uuids); + targets.removeIf(java.util.Objects::isNull); + + submit(() -> { + try { + // Anything already queued for these players has to land first, or it would be + // written back in behind the delete. + flush(); + + final Map tables = pluginTables(); + for (Map.Entry entry : tables.entrySet()) { + final String table = entry.getValue(); + for (int from = 0; from < targets.size(); from += IMPORT_PAGE) { + final List page = targets.subList(from, Math.min(targets.size(), from + IMPORT_PAGE)); + final List deletes = new ArrayList<>(page.size()); + for (String uuid : page) { + deletes.add(new Op.Clear(table, entry.getKey(), uuid)); + } + write(deletes); + } + } + progress.accept("&a&lDone. &fCleared the same data out of MySQL."); + } catch (SQLException err) { + fail("Unable to clear plugin data from MySQL", err); + progress.accept("&c&lHeads Up. &fThe files were reset, but MySQL still holds the old data: " + + err.getMessage()); + } + }); + } + + /** Turns one player's file into the rows that represent it. */ + private void collect(String uuid, FileConfiguration data, List batch) { + batch.add(new Op.Player(playerstable, uuid, + data.getString("Username"), data.getString("IP"), + data.getLong("Last-On"), data.getLong("Time-Played"))); + + final ConfigurationSection plugins = data.getConfigurationSection("Plugins"); + if (plugins == null) { + return; + } + + for (String pluginname : plugins.getKeys(false)) { + final ConfigurationSection section = plugins.getConfigurationSection(pluginname); + if (section == null) { + continue; + } + + final String table = Tables.data(settings.prefix, pluginname); + for (String path : section.getKeys(true)) { + final Object value = section.get(path); + if (value == null || value instanceof ConfigurationSection) { + continue; + } + if (tooLong(pluginname, path, "exporting")) { + continue; + } + batch.add(new Op.Set(table, pluginname, uuid, path, ValueCodec.encode(value))); + } + } + } + + /** Writes a batch straight through, bypassing the queue - used by import / export. */ + private void write(List batch) throws SQLException { + if (batch.isEmpty()) { + return; + } + final List ready = reduce(batch); + withConnection(connection -> { + ensureTables(connection, ready); + apply(connection, ready); + return null; + }); + written.addAndGet(ready.size()); + } + + // Import ------------------------------------------------------------------------------ + + /** + * Rebuilds the Data folder from MySQL. Used to stand a new server up from an existing + * database, or to pull a network's shared data down after a wipe. + */ + public void importAll(Consumer progress, Runnable then) { + final boolean queued = submit(() -> { + if (!longtask.compareAndSet(false, true)) { + progress.accept("&c&lBusy. &fAnother MySQL import or export is already running."); + finish(then); + return; + } + + final long start = System.currentTimeMillis(); + final boolean waspaused = plugin.isSavingPaused(); + + // Hold the file writer while the folder is rewritten underneath it. Queued changes + // simply wait, and land on the imported files once we are done. + plugin.setSavingPaused(true); + try { + final Map tables = pluginTables(); + int imported = 0; + String cursor = ""; + + while (true) { + final List page = playerPage(cursor); + if (page.isEmpty()) { + break; + } + + final Map>> values = valuesFor(tables, page); + + for (PlayerRow row : page) { + applyToFile(row, values.get(row.uuid), true); + imported++; + } + + cursor = page.get(page.size() - 1).uuid; + progress.accept("&7&oImported " + imported + " players..."); + } + + progress.accept("&a&lDone. &fImported &7&l" + imported + "&f players from MySQL in &7&l" + + (System.currentTimeMillis() - start) + "ms"); + } catch (SQLException err) { + fail("Import from MySQL failed", err); + progress.accept("&c&lFailed. &fMySQL rejected the import: " + err.getMessage()); + } finally { + plugin.setSavingPaused(waspaused); + longtask.set(false); + finish(then); + } + }); + + if (!queued) { + finish(then); + } + } + + /** + * Refreshes a single player's file from MySQL, for networks where a player's data follows + * them between servers. Their own file wins wherever it is newer, so a player who was last + * seen here never loses time to a stale row. + */ + public void pullPlayer(String uuid) { + submit(() -> { + try { + final Map tables = pluginTables(); + final PlayerRow row = playerRow(uuid); + if (row == null) { + plugin.debug("No MySQL record for " + uuid + " yet."); + return; + } + final Map>> values = + valuesFor(tables, Collections.singletonList(row)); + applyToFile(row, values.get(uuid), false); + plugin.debug("Refreshed " + uuid + "'s file from MySQL."); + } catch (SQLException err) { + fail("Unable to read " + uuid + " from MySQL", err); + } + }); + } + + /** plugin name (as used in the files) -> table holding its rows. */ + private Map pluginTables() throws SQLException { + return withConnection(connection -> { + final Map tables = new LinkedHashMap<>(); + try (PreparedStatement ps = connection.prepareStatement( + "SELECT `plugin`,`table_name` FROM `" + registrytable + "`"); + ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + tables.put(rs.getString(1), rs.getString(2)); + } + } + return tables; + }); + } + + private List playerPage(String after) throws SQLException { + return withConnection(connection -> { + final List rows = new ArrayList<>(); + try (PreparedStatement ps = connection.prepareStatement( + "SELECT `uuid`,`username`,`ip`,`last_on`,`time_played` FROM `" + playerstable + "`" + + " WHERE `uuid` > ? ORDER BY `uuid` ASC LIMIT " + IMPORT_PAGE)) { + ps.setString(1, after); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + rows.add(new PlayerRow(rs.getString(1), rs.getString(2), rs.getString(3), + rs.getLong(4), rs.getLong(5))); + } + } + } + return rows; + }); + } + + private PlayerRow playerRow(String uuid) throws SQLException { + return withConnection(connection -> { + try (PreparedStatement ps = connection.prepareStatement( + "SELECT `uuid`,`username`,`ip`,`last_on`,`time_played` FROM `" + playerstable + "` WHERE `uuid`=?")) { + ps.setString(1, uuid); + try (ResultSet rs = ps.executeQuery()) { + if (!rs.next()) { + return null; + } + return new PlayerRow(rs.getString(1), rs.getString(2), rs.getString(3), + rs.getLong(4), rs.getLong(5)); + } + } + }); + } + + /** uuid -> plugin name -> path -> encoded value, for one page of players. */ + private Map>> valuesFor(Map tables, List page) + throws SQLException { + final Map>> out = new LinkedHashMap<>(); + if (tables.isEmpty() || page.isEmpty()) { + return out; + } + + final StringBuilder placeholders = new StringBuilder(); + for (int i = 0; i < page.size(); i++) { + placeholders.append(i == 0 ? "?" : ",?"); + } + + for (Map.Entry entry : tables.entrySet()) { + final String pluginname = entry.getKey(); + final String table = entry.getValue(); + + try { + withConnection(connection -> { + // Ordered by path so a parent is always applied before its children. + try (PreparedStatement ps = connection.prepareStatement( + "SELECT `uuid`,`path`,`value` FROM `" + table + "` WHERE `uuid` IN (" + placeholders + + ") ORDER BY `uuid`,`path`")) { + for (int i = 0; i < page.size(); i++) { + ps.setString(i + 1, page.get(i).uuid); + } + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + out.computeIfAbsent(rs.getString(1), k -> new LinkedHashMap<>()) + .computeIfAbsent(pluginname, k -> new LinkedHashMap<>()) + .put(rs.getString(2), rs.getString(3)); + } + } + } + return null; + }); + } catch (SQLException err) { + // A registered plugin whose table was dropped by hand shouldn't stop the import. + plugin.getLogger().warning("Skipping MySQL table " + table + " for " + pluginname + ": " + err.getMessage()); + } + } + + return out; + } + + /** + * Writes one player's database state into their file. + * + * @param overwritecore true for a full import (the database is authoritative); false for a + * join refresh, where the local file wins if it has seen them since. + */ + private void applyToFile(PlayerRow row, Map> values, boolean overwritecore) { + final File folder = new File(plugin.getDataFolder(), File.separator + "Data"); + final File file = new File(folder, File.separator + row.uuid + ".yml"); + final FileConfiguration data = YamlConfiguration.loadConfiguration(file); + + final boolean newer = overwritecore || row.laston >= data.getLong("Last-On"); + + data.set("UUID", row.uuid); + if (newer) { + if (row.username != null) { + data.set("Username", row.username); + } + if (row.ip != null) { + data.set("IP", row.ip); + } + data.set("Last-On", row.laston); + } + data.set("Time-Played", overwritecore ? row.timeplayed : Math.max(row.timeplayed, data.getLong("Time-Played"))); + + if (values != null) { + for (Map.Entry> byplugin : values.entrySet()) { + for (Map.Entry value : byplugin.getValue().entrySet()) { + try { + data.set("Plugins." + byplugin.getKey() + "." + value.getKey(), + ValueCodec.decode(value.getValue())); + } catch (Exception err) { + plugin.getLogger().warning("Unreadable MySQL value for " + row.uuid + " at " + + byplugin.getKey() + "." + value.getKey() + ": " + err.getMessage()); + } + } + } + } + + try { + FileStore.save(data, file); + plugin.indexName(row.username, row.uuid); + } catch (Exception err) { + plugin.getLogger().warning("Unable to write " + file.getName() + " during a MySQL import: " + err.getMessage()); + } + } + + private File[] dataFiles() { + final File folder = new File(plugin.getDataFolder(), File.separator + "Data"); + final File[] files = folder.isDirectory() + ? folder.listFiles((dir, name) -> name.toLowerCase(Locale.ROOT).endsWith(".yml")) + : null; + return files == null ? new File[0] : files; + } + + // Plumbing ---------------------------------------------------------------------------- + + /** @return false if the writer thread is gone, in which case the task never runs. */ + private boolean submit(Runnable task) { + final ScheduledExecutorService service = worker; + if (service == null) { + plugin.getLogger().warning("MySQL isn't running; that request was ignored."); + return false; + } + try { + service.execute(task); + return true; + } catch (RejectedExecutionException shuttingdown) { + plugin.debug("MySQL task rejected, the plugin is shutting down."); + return false; + } + } + + /** Runs a caller's follow-up work without letting it take the writer thread down. */ + private void finish(Runnable then) { + if (then == null) { + return; + } + try { + then.run(); + } catch (Throwable err) { + plugin.getLogger().warning("Error after a MySQL task finished: " + err); + } + } + + private interface SqlTask { + T run(Connection connection) throws SQLException; + } + + private T withConnection(SqlTask task) throws SQLException { + final ConnectionPool open = pool; + if (open == null) { + throw new SQLException("The MySQL connection is not open."); + } + + final Connection connection = open.borrow(); + boolean healthy = false; + try { + final T result = task.run(connection); + healthy = true; + return result; + } finally { + if (healthy) { + open.release(connection); + } else { + open.discard(connection); + } + } + } + + private void fail(String what, Throwable err) { + lasterror = err.getMessage(); + if (quiet()) { + plugin.getLogger().warning(what + ": " + err.getMessage()); + if (plugin.isDebug()) { + err.printStackTrace(); + } + } + } + + /** True at most once a minute, so a database that is down doesn't flood the console. */ + private boolean quiet() { + final long now = System.currentTimeMillis(); + if (now < quietuntil) { + return false; + } + quietuntil = now + LOG_MUTE_MS; + return true; + } + + private String resolveDriver() { + for (String candidate : MYSQL_DRIVERS) { + if (load(candidate)) { + return candidate; + } + } + return load(MARIADB_DRIVER) ? MARIADB_DRIVER : null; + } + + private boolean load(String driver) { + try { + Class.forName(driver); + return true; + } catch (ClassNotFoundException | LinkageError missing) { + return false; + } + } + + private String url(String driver) { + // MariaDB's own connector dropped the jdbc:mysql: scheme in 3.x. + final String scheme = MARIADB_DRIVER.equals(driver) ? "jdbc:mariadb://" : "jdbc:mysql://"; + return scheme + settings.host + ":" + settings.port + "/" + settings.database; + } + + private Properties properties() { + final Properties props = new Properties(); + props.setProperty("user", settings.username); + props.setProperty("password", settings.password); + props.setProperty("useSSL", Boolean.toString(settings.usessl)); + props.setProperty("useUnicode", "true"); + props.setProperty("characterEncoding", "utf8"); + // Turns each JDBC batch into a single multi-row statement instead of one per row. + props.setProperty("rewriteBatchedStatements", "true"); + props.setProperty("connectTimeout", Long.toString(settings.connecttimeout * 1000L)); + props.setProperty("socketTimeout", Long.toString(Math.max(30, settings.connecttimeout * 3L) * 1000L)); + + for (String pair : settings.extraproperties.split("&")) { + final int split = pair.indexOf('='); + if (split > 0) { + props.setProperty(pair.substring(0, split).trim(), pair.substring(split + 1).trim()); + } + } + + return props; + } + + /** One row of the players table. */ + private static final class PlayerRow { + private final String uuid; + private final String username; + private final String ip; + private final long laston; + private final long timeplayed; + + private PlayerRow(String uuid, String username, String ip, long laston, long timeplayed) { + this.uuid = uuid; + this.username = username; + this.ip = ip; + this.laston = laston; + this.timeplayed = timeplayed; + } + } +} diff --git a/src/com/zachduda/puuids/storage/Op.java b/src/com/zachduda/puuids/storage/Op.java new file mode 100644 index 0000000..c759e6c --- /dev/null +++ b/src/com/zachduda/puuids/storage/Op.java @@ -0,0 +1,196 @@ +package com.zachduda.puuids.storage; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Types; + +/** + * One queued change, waiting to be sent to MySQL. + *

+ * A flush coalesces the queue before sending it (see {@code MySQLStorage#reduce}) and then + * groups what is left by {@link #batchKey()}, so a busy queue costs a couple of round trips + * rather than one per change. + */ +abstract class Op { + + /** + * Index-friendly cap on a data path. Deeper than anything a plugin sensibly stores, and + * short enough that (uuid, path) stays inside InnoDB's index limit on utf8mb4. + */ + static final int MAX_PATH = 191; + + final String table; + /** The plugin name as written in the .yml files, or null for puuids' own player table. */ + final String plugin; + final String uuid; + + private Op(String table, String plugin, String uuid) { + this.table = table; + this.plugin = plugin; + this.uuid = uuid; + } + + /** Operations sharing this key, and sitting next to each other, are batched together. */ + abstract String batchKey(); + + abstract String sql(); + + abstract void bind(PreparedStatement ps) throws SQLException; + + /** + * Escapes a path for use as a LIKE prefix. Data paths routinely contain underscores, which + * LIKE would otherwise treat as "any character" and delete a sibling's rows with. + */ + private static String childPattern(String path) { + final StringBuilder pattern = new StringBuilder(path.length() + 4); + for (int i = 0; i < path.length(); i++) { + final char c = path.charAt(i); + if (c == '!' || c == '%' || c == '_') { + pattern.append('!'); + } + pattern.append(c); + } + return pattern.append(".%").toString(); + } + + /** puuids' own record of a player: name, address, last seen and total play time. */ + static final class Player extends Op { + private final String username; + private final String ip; + private final long laston; + private final long timeplayed; + private final long updated; + + Player(String table, String uuid, String username, String ip, long laston, long timeplayed) { + super(table, null, uuid); + this.username = username; + this.ip = ip; + this.laston = laston; + this.timeplayed = timeplayed; + this.updated = System.currentTimeMillis(); + } + + String batchKey() { + return "player"; + } + + String sql() { + return "INSERT INTO `" + table + "` (`uuid`,`username`,`ip`,`last_on`,`time_played`,`updated`)" + + " VALUES (?,?,?,?,?,?)" + + " ON DUPLICATE KEY UPDATE `username`=VALUES(`username`),`ip`=VALUES(`ip`)," + + "`last_on`=VALUES(`last_on`),`time_played`=VALUES(`time_played`),`updated`=VALUES(`updated`)"; + } + + void bind(PreparedStatement ps) throws SQLException { + ps.setString(1, uuid); + if (username == null) { + ps.setNull(2, Types.VARCHAR); + } else { + ps.setString(2, username); + } + if (ip == null) { + ps.setNull(3, Types.VARCHAR); + } else { + ps.setString(3, ip); + } + ps.setLong(4, laston); + ps.setLong(5, timeplayed); + ps.setLong(6, updated); + } + } + + /** A single value belonging to one plugin. */ + static final class Set extends Op { + final String path; + private final String value; + private final long updated; + + Set(String table, String plugin, String uuid, String path, String value) { + super(table, plugin, uuid); + this.path = path; + this.value = value; + this.updated = System.currentTimeMillis(); + } + + String batchKey() { + return "set:" + table; + } + + String sql() { + return "INSERT INTO `" + table + "` (`uuid`,`path`,`value`,`updated`) VALUES (?,?,?,?)" + + " ON DUPLICATE KEY UPDATE `value`=VALUES(`value`),`updated`=VALUES(`updated`)"; + } + + void bind(PreparedStatement ps) throws SQLException { + ps.setString(1, uuid); + ps.setString(2, path); + ps.setString(3, value); + ps.setLong(4, updated); + } + } + + /** + * Clears out what used to live under a path. + *

+ * With {@code includeself} it is the mirror of setting null on file - the value and + * everything nested below it go. Without, only the nested values go: that is what runs + * before a value is re-written, so a path that used to hold a whole section can't leave + * orphaned children behind when it becomes a single value. + */ + static final class Remove extends Op { + final String path; + final boolean includeself; + + Remove(String table, String plugin, String uuid, String path, boolean includeself) { + super(table, plugin, uuid); + this.path = path; + this.includeself = includeself; + } + + String batchKey() { + return (includeself ? "removetree:" : "removechildren:") + table; + } + + String sql() { + if (includeself) { + return "DELETE FROM `" + table + "` WHERE `uuid`=? AND (`path`=? OR `path` LIKE ? ESCAPE '!')"; + } + return "DELETE FROM `" + table + "` WHERE `uuid`=? AND `path` LIKE ? ESCAPE '!'"; + } + + void bind(PreparedStatement ps) throws SQLException { + ps.setString(1, uuid); + if (includeself) { + ps.setString(2, path); + ps.setString(3, childPattern(path)); + } else { + ps.setString(2, childPattern(path)); + } + } + + /** Whether executing this would also delete the row at {@code other}. */ + boolean covers(String other) { + return (includeself && other.equals(path)) || other.startsWith(path + "."); + } + } + + /** Drops everything one plugin has stored for one player. */ + static final class Clear extends Op { + + Clear(String table, String plugin, String uuid) { + super(table, plugin, uuid); + } + + String batchKey() { + return "clear:" + table; + } + + String sql() { + return "DELETE FROM `" + table + "` WHERE `uuid`=?"; + } + + void bind(PreparedStatement ps) throws SQLException { + ps.setString(1, uuid); + } + } +} diff --git a/src/com/zachduda/puuids/storage/Tables.java b/src/com/zachduda/puuids/storage/Tables.java new file mode 100644 index 0000000..8365e2d --- /dev/null +++ b/src/com/zachduda/puuids/storage/Tables.java @@ -0,0 +1,87 @@ +package com.zachduda.puuids.storage; + +import java.util.Locale; + +/** + * Builds the table names, and the DDL for them. + *

+ * Every plugin that stores data gets its own table, so a server owner can look at + * {@code puuids_data_myplugin} and see exactly that plugin's rows - and drop the table when the + * plugin is gone without touching anybody else's data. Plugin names come from another author's + * plugin.yml, so they are reduced to plain identifier characters here: the result is + * interpolated into a statement, and nothing but this method is allowed to produce a table name. + */ +final class Tables { + + /** MySQL's identifier limit; the sanitized plugin name has to fit inside it with the prefix. */ + private static final int MAX_IDENTIFIER = 64; + + private Tables() { + } + + static String players(String prefix) { + return prefix + "players"; + } + + /** Maps the plugin name as it appears in the .yml files to the table holding its rows. */ + static String registry(String prefix) { + return prefix + "plugins"; + } + + static String data(String prefix, String plugin) { + final String base = prefix + "data_"; + return base + sanitize(plugin, MAX_IDENTIFIER - base.length()); + } + + private static String sanitize(String plugin, int room) { + String cleaned = plugin == null ? "" : plugin.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9_]", "_"); + + if (cleaned.isEmpty()) { + cleaned = "unknown"; + } + + if (cleaned.length() > room) { + /* + Two plugins whose names only differ past the cut-off would otherwise share a table, + so the tail is replaced by a hash of the full name rather than simply dropped. + */ + final String hash = Integer.toHexString(cleaned.hashCode()); + final int keep = Math.max(1, Math.min(cleaned.length(), room - hash.length() - 1)); + cleaned = cleaned.substring(0, keep) + "_" + hash; + } + + return cleaned; + } + + static String createPlayers(String table) { + return "CREATE TABLE IF NOT EXISTS `" + table + "` (" + + "`uuid` CHAR(36) NOT NULL," + + "`username` VARCHAR(16) DEFAULT NULL," + + "`ip` VARCHAR(45) DEFAULT NULL," + + "`last_on` BIGINT NOT NULL DEFAULT 0," + + "`time_played` BIGINT NOT NULL DEFAULT 0," + + "`updated` BIGINT NOT NULL DEFAULT 0," + + "PRIMARY KEY (`uuid`)," + + "KEY `idx_username` (`username`)" + + ") ENGINE=InnoDB ROW_FORMAT=DYNAMIC DEFAULT CHARSET=utf8mb4"; + } + + static String createRegistry(String table) { + return "CREATE TABLE IF NOT EXISTS `" + table + "` (" + + "`plugin` VARCHAR(64) NOT NULL," + + "`table_name` VARCHAR(64) NOT NULL," + + "`updated` BIGINT NOT NULL DEFAULT 0," + + "PRIMARY KEY (`plugin`)" + + ") ENGINE=InnoDB ROW_FORMAT=DYNAMIC DEFAULT CHARSET=utf8mb4"; + } + + static String createData(String table) { + return "CREATE TABLE IF NOT EXISTS `" + table + "` (" + + "`uuid` CHAR(36) NOT NULL," + + "`path` VARCHAR(" + Op.MAX_PATH + ") NOT NULL," + + "`value` MEDIUMTEXT," + + "`updated` BIGINT NOT NULL DEFAULT 0," + + "PRIMARY KEY (`uuid`,`path`)" + + ") ENGINE=InnoDB ROW_FORMAT=DYNAMIC DEFAULT CHARSET=utf8mb4"; + } +} diff --git a/src/com/zachduda/puuids/storage/ValueCodec.java b/src/com/zachduda/puuids/storage/ValueCodec.java new file mode 100644 index 0000000..fbec777 --- /dev/null +++ b/src/com/zachduda/puuids/storage/ValueCodec.java @@ -0,0 +1,39 @@ +package com.zachduda.puuids.storage; + +import org.bukkit.configuration.InvalidConfigurationException; +import org.bukkit.configuration.file.YamlConfiguration; + +/** + * Turns a single config value into text for a database column and back again. + *

+ * Values are stored as a one-key YAML document rather than as a plain string so the type + * survives the round trip: an int comes back an int, a list comes back a list, and anything + * Bukkit knows how to serialize - ItemStacks in particular - comes back as itself. That is the + * same representation the .yml files already use, so nothing is lost by mirroring through it. + */ +public final class ValueCodec { + + private static final String KEY = "v"; + + private ValueCodec() { + } + + public static String encode(Object value) { + final YamlConfiguration holder = new YamlConfiguration(); + holder.set(KEY, value); + return holder.saveToString(); + } + + /** + * @return the decoded value, or null if the column held something this server can't read + * (a value written by a plugin that is no longer installed, for instance). + */ + public static Object decode(String encoded) throws InvalidConfigurationException { + if (encoded == null || encoded.isEmpty()) { + return null; + } + final YamlConfiguration holder = new YamlConfiguration(); + holder.loadFromString(encoded); + return holder.get(KEY); + } +} diff --git a/src/config.yml b/src/config.yml index f4a5487..f47938f 100644 --- a/src/config.yml +++ b/src/config.yml @@ -14,6 +14,47 @@ Settings: Debug: false Messages: No-Permission: '&c&lSorry! &fYou are not able to do that.' +# Optionally mirror everything in the Data folder into MySQL. +# +# The .yml files stay in charge: every read still comes off disk, so this can never slow a +# plugin down or fail because the database is busy. Each write is copied up in the background +# instead, giving you one place to back up, query, or share between the servers on a network. +# +# One table is created per plugin (puuids_data_), alongside puuids_players for +# usernames / IPs / play time and puuids_plugins listing which table belongs to which plugin. +# +# Your server needs a MySQL or MariaDB JDBC driver on its classpath. Most Spigot and Paper +# builds ship one; if yours doesn't, puuids says so in the console and keeps saving to file. +MySQL: + Enabled: false + Host: localhost + Port: 3306 + Database: minecraft + Username: root + Password: '' + # Every table puuids creates starts with this. Letters, numbers and underscores only. + Table-Prefix: 'puuids_' + Use-SSL: false + # Passed straight to the driver, e.g. 'allowPublicKeyRetrieval=true&serverTimezone=UTC'. + # MySQL 8 with caching_sha2_password over an unencrypted link needs the first of those. + Extra-Properties: '' + # How long to wait on the database before giving up. The server start-up waits on this once. + Connection-Timeout-Seconds: 10 + Pool-Size: 3 + # How often queued changes are sent, in milliseconds. + Flush-Rate-Ms: 1000 + # If the database goes away, changes wait in memory up to this many before the oldest are + # dropped. Nothing is lost from your files - run /puuids mysql export to rebuild afterwards. + Max-Queued-Writes: 100000 + # Rebuild the Data folder from MySQL at start-up. For a server whose data lives in the + # database (a fresh network member, or one being restored) - it overwrites local files. + Import-On-Startup: false + # Push the whole Data folder up at start-up. Handy the first time you switch MySQL on; + # /puuids mysql export does the same thing on demand. + Export-On-Startup: false + # Refresh a player's file from MySQL as they join, for networks where their data follows them + # between servers. Their local file wins wherever it has seen them more recently. + Sync-On-Join: false Advanced: # Only mess with the following if you are sure as to what you're doing. Save-Rate-Ticks: 10 diff --git a/src/plugin.yml b/src/plugin.yml index f952576..eb63242 100644 --- a/src/plugin.yml +++ b/src/plugin.yml @@ -2,7 +2,7 @@ name: PUUIDs main: com.zachduda.puuids.Main website: https://zachduda.com/puuids authors: [zach_attack] -version: 4.0.0 +version: ${project.version} folia-supported: true api-version: 1.13 softdepend: [ Essentials ]