Probably time to give up on custom objects for networking. It just doesn't seem to work out well.

GeneralMods/StardustCore/Objects/CoreObject.cs

@@ -573,7 +573,13 @@ namespace StardustCore
 
             }
 
+
+            StardustCore.ModCore.SerializationManager.trackedObjectList.Remove(this);
+            this.thisLocation.removeObject(this.TileLocation, false);
+
             this.thisLocation = null;
+            this.locationsName = "";
+
             base.performRemoveAction(tileLocation, environment);
         }