From 957e1334dfcfc73d6578865841ab52a8e8a0c562 Mon Sep 17 00:00:00 2001 From: Obique PSWG Date: Sat, 25 Jul 2015 15:15:58 -0500 Subject: [PATCH] Created the new file type 'sdb' with intelligent loading into a sqlite database --- serverdata/static/spawns.db | Bin 0 -> 35840 bytes serverdata/static/spawns.sdb | 31 ++++ serverdata/static/types.sdb | 23 +++ serverdata/static_spawns.db | Bin 11264 -> 0 bytes src/main/ProjectSWG.java | 2 +- .../server_info/RelationalServerData.java | 167 ++++++++++++++++++ src/services/spawn/StaticService.java | 10 +- 7 files changed, 231 insertions(+), 2 deletions(-) create mode 100644 serverdata/static/spawns.db create mode 100644 serverdata/static/spawns.sdb create mode 100644 serverdata/static/types.sdb delete mode 100644 serverdata/static_spawns.db diff --git a/serverdata/static/spawns.db b/serverdata/static/spawns.db new file mode 100644 index 0000000000000000000000000000000000000000..b6deebec98646af60caf77527b2a6d6bdbfa1c79 GIT binary patch literal 35840 zcmeHQ3v65E75*Pi?4)`0ku+@siQ7{8ppd4o^pO&`G?X+>QqnYQ(uaHFIEjfLiS0B^ zp<{m}&@{HOAx(@m6eyz<#x}NQL%=q%0n?;uLX%(vRPh)R6NS+scK&<)J9d2C z#J>Mc$L1^DIQF&Af6nne-+#XIpL@OgW~VpevG)W+ekEdGgDePR+L7=5>Z^Rr8rPukwt9Qp)@{yud&;-jmuV6Mir-^ztKZwU z((Y5j5tr9L5DY~;-S&p&w)!3Qt;>5CTFaI!!EBcL{%}Nzc-^kZ$bcst|9?)xsPRYa z%e*~3>In8-tqo0ETRZHH^&Kni@Tw{%Hc-u?+_J_c7 zuIl(_x$q5I*wV$;vZYJ$XkomX;Q?hhAoSFp5Ise;L#n0;I<2(3`@FtxSImOumNt9y z9;Xuycziz1s`1y3*unL1Y$W=4H2T=*QM$c>UiI1X98j0g3;eWOL@&|{WHWvT--EZK z*U)j~I<>E|Ez=H+2Fr_gG(5)##CT!Q+>DmZ!iJrS2X=<|WP!B9-#gx9&mu@(36 z?RZ^OMaQC-`1X5TL zH@_zcC)C=p<^5dhdv$LfePhSzl3N`+bJRf=chIP8TapW=EISXh1XW|Q8*+R2)MQ*$3M^iF}(|I#z!O@u#S=99kEY~L8AvCpsmGxLdNuY9^85Kc~XPkrxhrI&XC7wJ3fMciZ9Tw z(HGGg{)Xu0h&zR1gu2%~456e}W4e^EM~vwW9QKAeXLogZ6n8MdwVk;|CxjB*b15t+ z-jp#E6ged#L7|=%QwD;9cqhhA%QSFcMy<~IGRfy-CtH3dql?pte}#D)G6Vsep4v&# zsMUj#u+O?aBM8XdNQaZ+IB84~9O8f)$u*fFfMpHsPZ}}d3>UztVFw?J>Ul=^T!d3R z#EE0@VF*lHhQ8rRelbk&^ac7=+LoRFf&6FZe{e#IGp#2(|HtL5U+no`&FAv*5vcz~ z5WbTPkT>yrcoWY7@UvPLke&ZiG6940KY0uljZYElXmI>VlpTqQNML`L66jCJPfM$) zJLyvivyA$Y*sNc&)56M9}TMDxY9*JZi=4z6|p}Osc0Yt*A&hVpRzWG67-4i#*Kd#PfuVs9szG9s9gn@ehb0`Trux>5XnGDHD?f|F&ji!a8{VUx2C+Jx2#=CHVunldQ#W;k&T|y@S37 z2PSTPEwHLy4(qenmJ@}zsjYU$mVLEKiAzhKxZ>J>=U#_pU|5oHRKgl4ta_Fh)mKz;aZ;O{MwP67 zD^6TnqfsTR$8?n3X~ToFV4>b-iGF31dlH&Z-0BPwsvfIQSGaq|)@bJMYET<1@@$wS z%S*qp;sV}$<4qtD7uujXK}byVq~hmM-pQ67;683>Ow8nlj`xg!r8d3*zwE+T^|#e? z@#L->Ra5o5P|`Ow_wx^(^`cosw6#pl(r9YK?Tl}d*$xI9X_C>_{|_O0njWB~)q1bVxnh4gEPdmpgDiL@`c|;`AM;LB$1u5&4~cr67NL;HboCq$k4B9HWuG zS`Ut98&xt9j%FEEQWuV9%7vq1HAZE^Q4vU#0Y`vna^a{zHB}-U@%Sh7RYcFjpX`?P zK-L3aY!8$Yx+EvM5~ub>G z0PjMdpi^k;6kO4Jg9=Id{i^9t@Dux2C=i%!{Pv)QmmCUn`3ciA*t5&m(E}$=Hg3#Zm}7#lPA&cwL{HIuG>1G*oUjJ)0KO4jM2{n73h_VLTbvuX zwl-+OPv3N5vTy-9yR#-L&6c?|;e&?tYLFB#B8`g55!f*|PP;y-IX_r)VcFhWg(SS%zQ3x8tkOZ;?MC_>Dj5TxdAO zN}Y7t4QGZalg>)RqEjZF6^1?HxLUh(aQv#V=%pQvS7x3kii}HSU?&@|$Xu96xP-7y zXSvjzaDl{}aK1W!NXZH3X^3w|PFSXbwTzsQ$G;W7HX;6vhk91RW%ocf{!?ElF37cj zBmq$k|BvxsfEM!Wf1Aj^$qCYq-^Y*QgSY_w9EIV)t{xS}Waj7|6=RJr_IAt8(cKzi z>)vu8eHPCbd%HxgDkVo(jOOTvR0H+1-SiEdnEl4pYF)x^DLMKx|A+PeC!qdU0QJ9K{29IYA6p019Tevvs1v{@%B@+vFeMXhkE7&#lN-e=@@xLF@Gt@)p zk!OiQa`7p=8xvUn+ixs~b>1d*7PuAMX-1{ysO(u_#GF~+h$FSiFN6O;KVZ)+R z7Egu@dnEOWT_p2*M{@BboVhT;OaxQubcUics+0va9{&YsIimN{c4{L}5jR;oy#&8Y3W5###!4 zwqCi5DGaVz;rB9=$kc8ZbCndP_GPd*g?f{(D)4*P9a9Z7pQ&l{e+#1Dp=-%`@^#|C z|HLP7JGP-mP>TruDL;0wIp)DVn=JwFQx(e@w7SY`(O}TxnyIv8#An4aA%7<;ii(rb zM8OJGl#oKoShT&>3|7YA=9IGfC|Ev^P7`?<Ir6b+8h{lWHn@rp13NqCcVCw1oVE_{m)S z0^SGle;TzWhV|5*b+Blwizz4$O)t_=e5u(e&Rb%p1;v{fG{s0IXS^*&eu0fGHmc?X za<<5*n)>8yp=5GqSAktBIjaPb63Cer|4{!wOM9XI_d9YMsl?CWUYw84$f*Cfu(i@5 ze2qyEzS>L(uQm42vQw7WSe0R?q^_|sXlc!qB_$Wx7&ND5DouxTt(42Tu93;P zR;a-*gLCotFF;EWeTeRb`+q-54v>8O4BYwi3iJ{hf&+YOVh+i;U1tt%UoRQAx5&rs zS~_hBdcOw#o~jSOySktrwh1F`ro_c zn`AwH6Y77p=#S`IQ@Z}w$=U_bNm)5zd(_iOcV%aCpyNv?tZmvbez!1P4#u}~)8%7) z3%gl52is*52Wv4*H!e{|Yc|Z2QkBsn4F6ti)hcd||sjzcf2R2=8stkQA3i5)Pi=7iGPexqvYm)7=4rf8ii z*h?*~b%00-rL`8e7hEWpqJ2rVRw6~y;=ha6|Mo)v?>V>&@FM&t=>I84=TMi*{tf== zWIn_2bqP%GH;lWfG5r?9vQuNa*RWGAQCEY`(@aT{4&{Q5(@dr5P_7=il*=QNa&@bb zFNbnPFD5aCa=~RKk|-CC{{r}VF8U+dPCg^2$#z2UllTxWLeHZR|J|-8c0@8{56g$_ z!?GZINIGOkWJ9)-K|86;sv0zu8my`fsy6B~7r9ln0b#lv#uenI>+3x^*}PQaLO?p> z@|(iAwE4eXt^ZL1_y5>|-^R!B_2@(NL)4z>)bAXnQc-@CNJRNjtb&CUlpl>YqMCy8 z(2^<%dYFt7fSig=OV_RYF;wzOYU zvDC~O|DCbe-QA|}GYJ;QOoPA0XW{Xmz5mB$KiSPtAbbCh?EODnI=_sOa8F1c|BzQ8 z`hB{ad`uo82T3vh6}*t$%<@3vD0Ho6T2M%XBt6oPdO4LgQIT9H?G!fq)AOC46m`rDK zGa~dvCgQ)Gh#Wc$pJRyx>ydN0h;bP`_r&krcPwG`K5TZuH>hNKow&YgI1o82pBA9u>QCvH9&Jr$Qs!;g%Q9^5xR85xUDf-#n-6-Pro1H|7w zwiBaFenx&+o6o^~7Nw%D7|&vbDg##Rvp-Q71YZk$VS1U3hu{2Rx6P}`K$C%u&p=Bn z0Vgqim$}+%{%m}&n;te9*nkX#`e_y`(+|l>Jvv>4y$|e2qIqdrQKDK}Qso)87f=g| zs*6VJ&tNM-+4@iDGNU6%FPk?TkejAUO$M3_*clMI;3*Ia(^IGw#}IxLpT*0hmn6xb z$lquOJ&0rUO}jwz-i?8w5PigvbPGADB&*S6MlV^ZH(k{AoV+6Y4r00xT$!@r46MN6 zO}0dN*5(~7aSss}z5E4B(x#O6F=4WY(yT-88cW51k7^E2Yh=<=epXV=#YNQ9vOL3- zwG6hnIGI#bID}^;%T(?p?_i?0`|14ZDm9yj)}ls+0xslmU`wM-l=m6r3<|X9&^nia zbG4*ec^phj`K-tr*)Rt7cDRV~VX6`1jfU7j zdv${7FcmUT(26FdxdInq_hKbt#c5eeDtRMp5@Fxsc88sFMcA&TBsHq(YB8x7RaxUe ztV_(x3*uZR51TwOB_&ySmKd6UcoTih0hX1tbg5LzRuS@IR)PwdnpSFKm<(@rk*}HO zs-k8@Fh?(H8uy1KHm083(MC%S^{a?i=Dk4#xq@!>5?!$^w(z$opG__RXGS4$TGFgE zn#mu}XlzXE)YIeWmO3>a*y>yCc6c@;>1kyy1EXYavH7z&UzAiSuR|UePbl!k_uhu1 z^aIpWm)dA2O*(YeuxW&^mHGmJ>WRWI9)|fJNAzVnO8!fpBX{C|<0qiucj&9SEMzYw z1OY!176O=%%3_vOWr>AsG+#(YVJVoC#e$?-;b*2q9y^SZkuUzpt`{z&rP*K*FLs0j zfq(^PjjgUq$K_lUio{u2hgqA;!D&aa*k9SskjuR$^2o>i#ufX?d+d52w~d4Ztc5Yg z0uVB>XLxhKm||1WrSmtj11*M<8zyN5TWDawE7(D>5=^zQ zNBt>wO_Fs($9%g!KX>ZEo+>ms_2910-=4zjW+ZmgNdu}1{g=;xqVe-#2#{8!BKjKr zHQh|!Aw_Z%{s1rHThK?){2g5BE|}iiVSusF;wf8LabxZ+MRU8!rlNWG)&+~0(`cI)E?oDKEh*7X|M z_o)qTqJ6za_LW_X+#=xlc9`jcukMKhf8r%&Hacowd51AYQssWr{Ue)=3?CFQapHN; zMa%fkd)ojJhXu%P9I?J7$Or&`g>9eCg z(lxY>>RN#AKsT;pkZWa}-4Y@*Mj-nEQ1`xD<{35V^;ch{#D-g5J> zCi+$@*K7`vUL-&ul0|Y3ze;bQllXu1T@s`3)NMfX#qW`Un62VvB^wVF=HQ@9F`kz( z9chktuoV)DbJsW>wcBH@;~pw-TkHV!I>b{HGpp4t8?8ZzRW1wB->;pG+>#_GhBIa) nN#4lJ>TolQzrdyJ|6cGugKFS;kCW@wT5M~OH4Jssp4-0x_GHo( diff --git a/src/main/ProjectSWG.java b/src/main/ProjectSWG.java index 8b2c2b484..2510a3b07 100644 --- a/src/main/ProjectSWG.java +++ b/src/main/ProjectSWG.java @@ -172,7 +172,7 @@ public class ProjectSWG { System.out.println("ProjectSWG: Terminated. Time: " + manager.getCoreTime() + "ms"); } - private static class CoreException extends RuntimeException { + public static class CoreException extends RuntimeException { private static final long serialVersionUID = 455306876887818064L; diff --git a/src/resources/server_info/RelationalServerData.java b/src/resources/server_info/RelationalServerData.java index c4435cad2..260feab6a 100644 --- a/src/resources/server_info/RelationalServerData.java +++ b/src/resources/server_info/RelationalServerData.java @@ -27,10 +27,177 @@ ***********************************************************************************/ package resources.server_info; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + public class RelationalServerData extends RelationalDatabase { + private static final String META_TABLE = "__server_table_metadata__"; + private final PreparedStatement getTableMetadata; + private final PreparedStatement updateTableMetadata; + private final PreparedStatement insertTableMetadata; + public RelationalServerData(String file) { super("org.sqlite.JDBC", "jdbc:sqlite:" + file); + /* + * GREATLY speeds up INSERT speeds. This makes it so SQLite will not + * wait until the previous write finishes before starting to write the + * next INSERT. All imports will only take milliseconds. A caveat of + * this is that if the power ever cuts or the application is force + * killed it may result in database corruption. + */ + updateQuery("PRAGMA synchronous=OFF"); + updateQuery("CREATE TABLE IF NOT EXISTS "+META_TABLE+" (table_name TEXT, last_imported INTEGER)"); + getTableMetadata = prepareStatement("SELECT * FROM "+META_TABLE+" WHERE table_name=?"); + updateTableMetadata = prepareStatement("UPDATE "+META_TABLE+" SET last_imported=? WHERE table_name=?"); + insertTableMetadata = prepareStatement("INSERT INTO "+META_TABLE+" (table_name, last_imported) VALUES (?, ?)"); + } + + public void linkTableWithSdb(String table, String sdbPath) throws FileNotFoundException { + File sdb = new File(sdbPath); + if (!sdb.isFile()) + throw new FileNotFoundException("SDB does not exist at path " + sdb); + long sdbModified = sdb.lastModified(); + long imported = getLastImported(table); + if (sdbModified > imported) { + updateQuery("DROP TABLE " + table); + importFromSdb(table, sdb); + updateLastImported(table, System.currentTimeMillis()); + } + } + + private long getLastImported(String table) { + synchronized (getTableMetadata) { + ResultSet set = null; + try { + getTableMetadata.setString(1, table); + set = getTableMetadata.executeQuery(); + if (set.next()) + return set.getLong("last_imported"); + } catch (SQLException e) { + e.printStackTrace(); + } finally { + if (set != null) { + try { + set.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + } + return -1; + } + } + + private void updateLastImported(String table, long lastImported) { + synchronized (updateTableMetadata) { + try { + updateTableMetadata.setLong(1, lastImported); + updateTableMetadata.setString(2, table); + if (updateTableMetadata.executeUpdate() > 0) + return; + } catch (SQLException e) { + e.printStackTrace(); + return; + } + } + synchronized (insertTableMetadata) { + try { + insertTableMetadata.setString(1, table); + insertTableMetadata.setLong(2, lastImported); + insertTableMetadata.executeUpdate(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + } + + private boolean importFromSdb(String table, File sdb) { + try { + processSdb(table, sdb); + System.out.println("Imported from sdb: " + sdb); + return true; + } catch (IOException e) { + e.printStackTrace(); + } catch (SQLException e) { + e.printStackTrace(); + } catch (NumberFormatException e) { + System.err.println("Invalid record in sdb. Aborting read of " + sdb + "!"); + } catch (IllegalArgumentException e) { + System.err.println("Invalid file format. Aborting read of " + sdb + "!"); + } + return false; + } + + private void processSdb(String table, File file) throws IOException, SQLException { + BufferedReader reader = new BufferedReader(new FileReader(file)); + String line = null; + String [] columnNames = null; + String [] columnTypes = null; + PreparedStatement insert = null; + try { + while ((line = reader.readLine()) != null) { + String [] parts = line.split("\t"); + if (columnNames == null) + columnNames = parts; + else if (columnTypes == null) { + columnTypes = parts; + createTable(table, columnNames, columnTypes); + insert = prepareStatement(createPreparedStatement(table, columnNames.length)); + } else + generateInsert(insert, columnTypes, parts); + } + if (insert != null) + insert.executeBatch(); + } finally { + reader.close(); + } + } + + private void createTable(String table, String [] names, String [] types) { + if (names.length != types.length) + throw new IllegalArgumentException("Names length and Types length mismatch"); + String sql = "CREATE TABLE "+table+" ("; + for (int i = 0; i < names.length; i++) { + if (i > 0) + sql += ", "; + sql += names[i] + " " + types[i]; + } + sql += ")"; + updateQuery(sql); + } + + private String createPreparedStatement(String table, int valueSize) { + String sql = "INSERT INTO " + table + " VALUES ("; + for (int i = 0; i < valueSize; i++) { + if (i > 0) + sql += ", "; + sql += "?"; + } + sql += ")"; + return sql; + } + + private void generateInsert(PreparedStatement insert, String [] types, String [] data) throws SQLException { + if (types.length != data.length) + throw new IllegalArgumentException("Types length and Data length mismatch"); + for (int i = 0; i < data.length; i++) { + if (types[i].startsWith("TEXT")) + insert.setString(i+1, data[i]); + else if (types[i].startsWith("REAL")) + insert.setDouble(i+1, Double.valueOf(data[i])); + else if (types[i].startsWith("INTEGER")) + insert.setLong(i+1, Long.valueOf(data[i])); + else + throw new SQLException("Data type unsupported by sdb/sqlite! Type: " + types[i]); + } + insert.addBatch(); } } diff --git a/src/services/spawn/StaticService.java b/src/services/spawn/StaticService.java index 2d4370d16..059cb35cf 100644 --- a/src/services/spawn/StaticService.java +++ b/src/services/spawn/StaticService.java @@ -27,6 +27,7 @@ ***********************************************************************************/ package services.spawn; +import java.io.FileNotFoundException; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; @@ -51,7 +52,14 @@ public class StaticService extends Service { this.databaseMutex = new Object(); this.objectManager = objectManager; - spawnDatabase = new RelationalServerData("serverdata/static_spawns.db"); + spawnDatabase = new RelationalServerData("serverdata/static/spawns.db"); + try { + spawnDatabase.linkTableWithSdb("static_spawns", "serverdata/static/spawns.sdb"); + spawnDatabase.linkTableWithSdb("static_types", "serverdata/static/types.sdb"); + } catch (FileNotFoundException e) { + e.printStackTrace(); + throw new main.ProjectSWG.CoreException("Unable to load sdb files for StaticService"); + } getSupportingStatement = spawnDatabase.prepareStatement(GET_SUPPORTING_SQL); }