Changeset 26:3b888f173cfb in rrlib_serialization-java

Timestamp:

08.06.2017 03:02:37 (22 months ago)

Author:

Max Reichardt <mreichardt@…>

Branch:

17.03

Phase:

public

Message:

Adapts to changes in rrlib_serialization and rrlib_rtti (Finroc17). These include versioned serialization and introduction of register auto-publishing mechanism.

Files:

• BinaryInputStream.java (modified) (14 diffs)
• BinaryOutputStream.java (modified) (9 diffs)
• BufferInfo.java (modified) (1 diff)
• ConstSource.java (deleted)
• FixedBuffer.java (modified) (1 diff)
• MemoryBuffer.java (modified) (3 diffs)
• PortDataListImpl.java (modified) (2 diffs)
• PublishedRegisters.java (added)
• Register.java (added)
• RegisterUpdate.java (added)
• SerializationInfo.java (added)
• StringOutputStream.java (modified) (2 diffs)
• StringSerializable.java (modified) (2 diffs)
• TypeEncoder.java (deleted)
• rtti/DataType.java (modified) (5 diffs)
• rtti/DataTypeBase.java (modified) (14 diffs)

BinaryInputStream.java

@@ -25 +25 @@
 import java.io.StringReader;
 
-import org.rrlib.logging.Log;
 import org.rrlib.serialization.compression.Compressible;
 import org.rrlib.serialization.compression.DataCompressor;
-import org.rrlib.serialization.rtti.DataTypeBase;
 import org.rrlib.xml.XMLDocument;
 import org.rrlib.xml.XMLNode;
@@ -57 +55 @@
     protected Source source = null;
 
-    /** Manager that handles, where the data blocks come from etc. */
-    protected ConstSource constSource = null;
-
     /** Current absolute buffer read position of buffer start - relevant when using Source; 64bit value, because we might transfer several GB over a stream */
     protected long absoluteReadPos = 0;
@@ -75 +70 @@
     protected int timeout = -1;
 
-    /** Data type encoding */
-    public enum TypeEncoding {
-        LocalUids, // use local uids. fastest. Can, however, only be used with streams encoded by this runtime.
-        Names, // use names. Can be decoded in any runtime that knows types.
-        Custom // use custom type codec
-    }
-
-    /** Data type encoding that is used */
-    protected TypeEncoding encoding;
-
-    /** Custom type encoder */
-    protected TypeEncoder customEncoder;
-
     /** ByteArrayInputStream helper for reading strings in Java */
     protected ByteArrayOutputStream baos = new ByteArrayOutputStream();
 
-    public BinaryInputStream() {
-        this(TypeEncoding.LocalUids);
-    }
-
-    public BinaryInputStream(ConstSource source_) {
-        this(source_, TypeEncoding.LocalUids);
-    }
-
-    public BinaryInputStream(TypeEncoding encoding) {
-        boundaryBuffer.buffer = boundaryBufferBackend;
-        this.encoding = encoding;
-    }
-
-    public BinaryInputStream(TypeEncoder encoder) {
-        boundaryBuffer.buffer = boundaryBufferBackend;
-        customEncoder = encoder;
-        encoding = TypeEncoding.Custom;
-    }
-
-    public BinaryInputStream(ConstSource source_, TypeEncoding encoding) {
-        boundaryBuffer.buffer = boundaryBufferBackend;
-        this.encoding = encoding;
-        reset(source_);
-    }
-
-    public BinaryInputStream(ConstSource source_, TypeEncoder encoder) {
-        boundaryBuffer.buffer = boundaryBufferBackend;
-        customEncoder = encoder;
-        encoding = TypeEncoding.Custom;
-        reset(source_);
-    }
-
-
-    /**
-     * @param source Source that handles, where the data blocks come from etc.
-     */
-    public BinaryInputStream(Source source_) {
-        boundaryBuffer.buffer = boundaryBufferBackend;
-        reset(source_);
-    }
-
-    /**
-     * @param source Source that handles, where the data blocks come from etc.
-     */
-    public BinaryInputStream(Source source_, TypeEncoder encoder) {
-        boundaryBuffer.buffer = boundaryBufferBackend;
-        customEncoder = encoder;
-        encoding = TypeEncoding.Custom;
-        reset(source_);
+    /** Info on source that created data currently read (at least the included revision is required for deserialization) */
+    private SerializationInfo serializationSource;
+
+    /** Replicated remote registers */
+    PublishedRegisters.RemoteRegister[] remoteRegisters;
+
+
+    public BinaryInputStream() {}
+
+    /**
+     * @param source Source to use
+     */
+    public BinaryInputStream(Source source) {
+        this(source, new SerializationInfo());
+    }
+
+    /**
+     * @param source Source to use
+     * @param sourceInfo Info on source that created data currently read (at least the included revision is required for deserialization)
+     */
+    public BinaryInputStream(Source source, SerializationInfo sourceInfo) {
+        reset(source, sourceInfo);
+    }
+
+    /**
+     * @param source Source to use
+     * @param sharedSerializationInfoFrom Serialization info (SerializationInfo and published registers) is taken from and shared with this input stream
+     */
+    public BinaryInputStream(Source source, BinaryInputStream sharedSerializationInfoFrom) {
+        reset(source, sharedSerializationInfoFrom);
+    }
+
+    /**
+     * @return Info on source that created data currently read (at least the included revision is required for deserialization)
+     */
+    public SerializationInfo getSourceInfo() {
+        return serializationSource;
     }
 
@@ -155 +125 @@
      */
     public void reset() {
-        if (source != null) {
-            source.reset(this, sourceBuffer);
-            directReadSupport = source.directReadSupport();
-            closed = false;
-        } else if (constSource != null) {
-            constSource.reset(this, sourceBuffer);
-            directReadSupport = constSource.directReadSupport();
-            closed = false;
-        }
+        source.reset(this, sourceBuffer);
+        directReadSupport = source.directReadSupport();
+        closed = false;
         curBuffer = sourceBuffer;
         absoluteReadPos = 0;
@@ -174 +138 @@
      * @param source New Source
      */
-    public void reset(ConstSource source) {
-        close();
-        this.source = null;
-        this.constSource = source;
-        reset();
+    public void reset(Source source) {
+        reset(source, new SerializationInfo());
     }
 
@@ -186 +147 @@
      *
      * @param source New Source
-     */
-    public void reset(Source source) {
+     * @param sourceInfo Info on source that created data currently read (at least the included revision is required for deserialization)
+     */
+    public void reset(Source source, SerializationInfo sourceInfo) {
         close();
         this.source = source;
-        this.constSource = null;
+        setSerializationSource(sourceInfo);
         reset();
+    }
+
+    /**
+     * Sets shared serialization source of this stream without changing sink
+     *
+     * @param sourceInfo Info on source that created data currently read (at least the included revision is required for deserialization)
+     */
+    public void setSerializationSource(SerializationInfo sourceInfo) {
+        this.serializationSource = sourceInfo;
+        if (sourceInfo.hasPublishedRegisters()) {
+            this.remoteRegisters = new PublishedRegisters.RemoteRegister[SerializationInfo.MAX_PUBLISHED_REGISTERS];
+            for (int i = 0; i < SerializationInfo.MAX_PUBLISHED_REGISTERS; i++) {
+                if (sourceInfo.getRegisterEntryEncoding(i) == SerializationInfo.RegisterEntryEncoding.PUBLISH_REGISTER_ON_CHANGE || sourceInfo.getRegisterEntryEncoding(i) == SerializationInfo.RegisterEntryEncoding.PUBLISH_REGISTER_ON_DEMAND) {
+                    this.remoteRegisters[i] = PublishedRegisters.get(i).createRemoteRegister();
+                }
+            }
+
+        } else {
+            this.remoteRegisters = null;
+        }
+    }
+
+
+    /**
+     * Use this object with different source.
+     * Current source will be closed.
+     *
+     * @param source New Source
+     * @param sharedSerializationInfoFrom Serialization info (SerializationInfo and published registers) is taken from and shared with this input stream
+     */
+    public void reset(Source source, BinaryInputStream sharedSerializationInfoFrom) {
+        close();
+        this.source = source;
+        this.serializationSource = sharedSerializationInfoFrom.serializationSource;
+        this.remoteRegisters = sharedSerializationInfoFrom.remoteRegisters;
+        reset();
+    }
+
+    /**
+     * Sets shared serialization info of this stream without changing source
+     *
+     * @param sharedSerializationInfoFrom Serialization info (SerializationInfo and published registers) is taken from and shared with this input stream
+     */
+    public void setSharedSerializationInfo(BinaryInputStream sharedSerializationInfoFrom) {
+        this.serializationSource = sharedSerializationInfoFrom.serializationSource;
+        this.remoteRegisters = sharedSerializationInfoFrom.remoteRegisters;
+    }
+
+    /**
+     * Sets shared serialization info of this stream without changing source
+     *
+     * @param sourceInfo Info on source that created data currently read
+     * @param remoteRegistersFrom Remote registers from this stream are used
+     */
+    public void setSharedSerializationInfo(SerializationInfo sourceInfo, BinaryInputStream remoteRegistersFrom) {
+        this.serializationSource = sourceInfo;
+        this.remoteRegisters = remoteRegistersFrom.remoteRegisters;
     }
 
@@ -201 +220 @@
             if (source != null) {
                 source.close(this, sourceBuffer);
-            } else if (constSource != null) {
-                constSource.close(this, sourceBuffer);
             }
         }
@@ -370 +387 @@
     protected void fetchNextBytes(int minRequired2) {
         assert(minRequired2 <= 8);
-        assert(source != null || constSource != null);
+        assert(source != null);
 
         // are we finished using boundary buffer?
@@ -397 +414 @@
             int initialSleep = 20; // timeout-related
             int slept = 0; // timeout-related
-            while (timeout > 0 && (!(source != null ? source.moreDataAvailable(this, sourceBuffer) : constSource.moreDataAvailable(this, sourceBuffer)))) {
+            while (timeout > 0 && (!source.moreDataAvailable(this, sourceBuffer))) {
                 initialSleep *= 2;
                 try {
@@ -410 +427 @@
 
         // read next block
-        if (source != null) {
-            source.read(this, sourceBuffer, minRequired2);
-        } else {
-            constSource.read(this, sourceBuffer, minRequired2);
-        }
+        source.read(this, sourceBuffer, minRequired2);
         assert(sourceBuffer.remaining() >= minRequired2);
         assert(sourceBuffer.position >= 0);
@@ -570 +583 @@
 
     /**
+     * @return sizeOf Size of integer in byte (1, 2, 4, or 8)
+     */
+    public long readInt(int sizeOf) {
+        switch (sizeOf) {
+        case 1:
+            return readByte();
+        case 2:
+            return readShort();
+        case 4:
+            return readInt();
+        case 8:
+            return readLong();
+        default:
+            throw new RuntimeException("Invalid size");
+        }
+    }
+
+
+    /**Info on source that created data currently read (at least the included revision is required for deserialization)
      * @return String/Line from stream (ends either at line delimiter or 0-character)
      */
@@ -702 +734 @@
         }
         //System.out.println("Not here");
-        return source != null ? source.moreDataAvailable(this, sourceBuffer) : constSource.moreDataAvailable(this, sourceBuffer);
+        return source.moreDataAvailable(this, sourceBuffer);
     }
 
@@ -724 +756 @@
     public void setTimeout(int timeout) {
         this.timeout = timeout;
-    }
-
-    /**
-     * @return Reads type from stream
-     */
-    public DataTypeBase readType() {
-        if (encoding == TypeEncoding.LocalUids) {
-            return DataTypeBase.getType(readShort());
-        } else if (encoding == TypeEncoding.Names) {
-            return DataTypeBase.findType(readString());
-        } else {
-            return customEncoder.readType(this);
-        }
     }
 
@@ -882 +901 @@
     }
 
-
+    /**
+     * @param registerUid Uid of register to serialize remote entry of
+     * @return Remote register entry of specified type
+     */
+    public PublishedRegisters.RemoteEntryBase<?> readRegisterEntry(int registerUid) throws Exception {
+        if (remoteRegisters == null) {
+            throw new Exception("BinaryInputStream: No shared serialization info set");
+        }
+        Register<?> localRegister = PublishedRegisters.get(registerUid).register;
+        int handle = (int)readInt(localRegister.getSizeOfHandle());
+        if (handle == -2) {
+            readRegisterUpdates();
+            handle = (int)readInt(localRegister.getSizeOfHandle());
+        }
+        if (handle == -1) {
+            return PublishedRegisters.getMinusOneElement();
+        }
+        return remoteRegisters[registerUid].get(handle);
+    }
+
+    /**
+     * @return Reads register updates
+     */
+    void readRegisterUpdates() throws Exception {
+
+        // Read register updates
+        if (getSourceInfo().getRevision() == 0) {
+            remoteRegisters[0].deserializeEntries(this);
+        } else {
+            byte uid = readByte();
+            while (uid != -1) {
+                if (uid >= remoteRegisters.length || uid < 0) {
+                    throw new RuntimeException("Invalid register uid " + uid);
+                }
+                remoteRegisters[uid].deserializeEntries(this);
+                uid = readByte();
+            }
+        }
+    }
 }

BinaryOutputStream.java

@@ -22 +22 @@
 package org.rrlib.serialization;
 
-import org.rrlib.serialization.rtti.DataTypeBase;
+import java.util.ArrayList;
+
 import org.rrlib.xml.XMLDocument;
 import org.rrlib.xml.XMLNode;
@@ -82 +83 @@
     private boolean directWriteSupport = false;
 
-    /** Data type encoding */
-    public enum TypeEncoding {
-        LocalUids, // use local uids. fastest. Can, however, only be decoded in this runtime.
-        Names, // use names. Can be decoded in any runtime that knows types.
-        Custom // use custom type codec
-    }
-
-    /** Data type encoding that is used */
-    private TypeEncoding encoding;
-
-    /** Custom type encoder */
-    private TypeEncoder customEncoder;
-
-    public BinaryOutputStream() {
-        this(TypeEncoding.LocalUids);
-    }
-
-    public BinaryOutputStream(Sink sink_) {
-        this(sink_, TypeEncoding.LocalUids);
-    }
-
-    /**
-     * @param encoding Data type encoding that is used
-     */
-    public BinaryOutputStream(TypeEncoding encoding) {
-        this.encoding = encoding;
-    }
-
-    public BinaryOutputStream(TypeEncoder encoder) {
-        customEncoder = encoder;
-        encoding = TypeEncoding.Custom;
-    }
-
-    /**
-     * @param sink_ Sink to write to
-     * @param encoding Data type encoding that is used
-     */
-    public BinaryOutputStream(Sink sink_, TypeEncoding encoding) {
-        this.encoding = encoding;
-        reset(sink_);
-    }
-
-    /**
-     * @param sink_ Sink to write to
-     * @param encoder Custom type encoder
-     */
-    public BinaryOutputStream(Sink sink_, TypeEncoder encoder) {
-        customEncoder = encoder;
-        encoding = TypeEncoding.Custom;
-        reset(sink_);
+    /** Info on target that serialization is created for */
+    private SerializationInfo serializationTarget;
+
+    /** Info on a single published registers */
+    class PublishedRegisterStatus {
+
+        /** Contains number of entries from register that were already written to stream */
+        final int[] elementsWritten = new int[SerializationInfo.MAX_PUBLISHED_REGISTERS];
+
+        /** Published entries of register that are updates on change */
+        int publishedOnChangedEntries;
+
+        /** Registers that are updated on change */
+        ArrayList<Register<?>> onChangeRegisters = new ArrayList<Register<?>>();
+    }
+
+    /** Info on published registers - possibly null */
+    PublishedRegisterStatus publishedRegisterStatus;
+
+
+    /**
+     * @param sink Sink to use
+     */
+    public BinaryOutputStream(Sink sink) {
+        this(sink, new SerializationInfo());
+    }
+
+    /**
+     * @param sink Sink to use
+     * @param serializationTarget Info on target that serialization is created for
+     */
+    public BinaryOutputStream(Sink sink, SerializationInfo serializationTarget) {
+        reset(sink, serializationTarget);
+    }
+
+    /**
+     * @param sink Sink to use
+     * @param sharedSerializationInfoFrom Serialization info (SerializationInfo and published registers) is taken from and shared with this output stream
+     */
+    public BinaryOutputStream(Sink sink, BinaryOutputStream sharedSerializationInfoFrom) {
+        reset(sink, sharedSerializationInfoFrom);
+    }
+
+
+    public BinaryOutputStream() {}
+
+    /**
+     * @return Info on target that serialization is created for
+     */
+    public SerializationInfo getTargetInfo() {
+        return serializationTarget;
     }
 
@@ -147 +149 @@
     public int remaining() {
         return buffer.remaining();
-    }
-
-    /**
-     * Use buffer with different sink (closes old one)
-     *
-     * @param sink New Sink to use
-     */
-    public void reset(Sink sink) {
-        close();
-        this.sink = sink;
-        reset();
     }
 
@@ -169 +160 @@
         bufferCopyFraction = (int)(buffer.capacity() * BUFFER_COPY_FRACTION);
         directWriteSupport = sink.directWriteSupport();
+    }
+
+    /**
+     * Use buffer with different sink (closes old one)
+     *
+     * @param sink New Sink to use
+     */
+    public void reset(Sink sink) {
+        reset(sink, new SerializationInfo());
+    }
+
+    /**
+     * Use buffer with different sink (closes old one)
+     *
+     * @param sink New Sink to use
+     * @param serializationTarget Info on target that serialization is created for
+     */
+    public void reset(Sink sink, SerializationInfo serializationTarget) {
+        close();
+        this.sink = sink;
+        setSerializationTarget(serializationTarget);
+        reset();
+    }
+
+    /**
+     * Sets shared serialization target of this stream without changing sink
+     *
+     * serializationTarget Info on target that serialization is created for
+     */
+    public void setSerializationTarget(SerializationInfo serializationTarget) {
+        this.serializationTarget = serializationTarget;
+        publishedRegisterStatus = serializationTarget.hasPublishedRegisters() ? new PublishedRegisterStatus() : null;
+        if (serializationTarget.hasPublishedRegisters()) {
+            for (int i = 0; i < SerializationInfo.MAX_PUBLISHED_REGISTERS; i++) {
+                if (serializationTarget.getRegisterEntryEncoding(i) == SerializationInfo.RegisterEntryEncoding.PUBLISH_REGISTER_ON_CHANGE) {
+                    publishedRegisterStatus.onChangeRegisters.add(PublishedRegisters.get(i).register);
+                }
+            }
+        }
+    }
+
+    /**
+     * Use buffer with different sink (closes old one)
+     *
+     * @param sink New Sink to use
+     * @param sharedSerializationInfoFrom Serialization info (SerializationInfo and published registers) is taken from and shared with this output stream
+     */
+    public void reset(Sink sink, BinaryOutputStream sharedSerializationInfoFrom) {
+        close();
+        this.sink = sink;
+        setSharedSerializationInfo(sharedSerializationInfoFrom);
+        reset();
+    }
+
+    /**
+     * Sets shared serialization info of this stream without changing sink
+     *
+     * @param sharedSerializationInfoFrom Serialization info (SerializationInfo and published registers) is taken from and shared with this output stream
+     */
+    public void setSharedSerializationInfo(BinaryOutputStream sharedSerializationInfoFrom) {
+        this.serializationTarget = sharedSerializationInfoFrom.serializationTarget;
+        this.publishedRegisterStatus = sharedSerializationInfoFrom.publishedRegisterStatus;
     }
 
@@ -377 +430 @@
 
     /**
-     * @param v 32 bit integer
-     */
-    public void writeInt(int v) {
-        ensureAdditionalCapacity(4);
-        buffer.buffer.putInt(buffer.position, v);
-        buffer.position += 4;
-    }
-
-    /**
-     * @param v 64 bit integer
-     */
-    public void writeLong(long v) {
-        ensureAdditionalCapacity(8);
-        buffer.buffer.putLong(buffer.position, v);
-        buffer.position += 8;
-    }
-
-    /**
-     * @param v 16 bit integer
-     */
-    public void writeShort(int v) {
-        ensureAdditionalCapacity(2);
-        buffer.buffer.putShort(buffer.position, (short)v);
-        buffer.position += 2;
-    }
-
-    /**
      * @param v 64 bit floating point
      */
@@ -412 +438 @@
     }
 
-    /**
-     * @param v 32 bit floating point
-     */
-    public void writeFloat(float v) {
-        ensureAdditionalCapacity(4);
-        buffer.buffer.putFloat(buffer.position, v);
-        buffer.position += 4;
-    }
 
     /**
@@ -447 +465 @@
 
     /**
+     * @param v 32 bit floating point
+     */
+    public void writeFloat(float v) {
+        ensureAdditionalCapacity(4);
+        buffer.buffer.putFloat(buffer.position, v);
+        buffer.position += 4;
+    }
+
+    /**
+     * @param v 32 bit integer
+     */
+    public void writeInt(int v) {
+        ensureAdditionalCapacity(4);
+        buffer.buffer.putInt(buffer.position, v);
+        buffer.position += 4;
+    }
+
+    /**
+     * @param v sizeof byte integer
+     * @param sizeof Size of integer (1, 2, 4, or 8)
+     */
+    public void writeInt(long v, int sizeof) {
+        switch (sizeof) {
+        case 1:
+            writeByte((byte)v);
+            break;
+        case 2:
+            writeShort((short)v);
+            break;
+        case 4:
+            writeInt((int)v);
+            break;
+        case 8:
+            writeLong(v);
+            break;
+        default:
+            throw new RuntimeException("Invalid size");
+        }
+    }
+
+    /**
+     * @param v 64 bit integer
+     */
+    public void writeLong(long v) {
+        ensureAdditionalCapacity(8);
+        buffer.buffer.putLong(buffer.position, v);
+        buffer.position += 8;
+    }
+
+    /**
+     * @param v 16 bit integer
+     */
+    public void writeShort(int v) {
+        ensureAdditionalCapacity(2);
+        buffer.buffer.putShort(buffer.position, (short)v);
+        buffer.position += 2;
+    }
+
+    /**
      * Print line to StreamBuffer.
      *
@@ -536 +613 @@
             write(inputStream.curBuffer.buffer, inputStream.curBuffer.position, inputStream.curBuffer.remaining());
             inputStream.curBuffer.position = inputStream.curBuffer.end;
-        }
-    }
-
-    /**
-     * @param type Data type to write/reference (using encoding specified in constructor)
-     */
-    public void writeType(DataTypeBase type) {
-        type = type == null ? DataTypeBase.NULL_TYPE : type;
-
-        if (encoding == TypeEncoding.LocalUids) {
-            writeShort(type.getUid());
-        } else if (encoding == TypeEncoding.Names) {
-            writeString(type.getName());
-        } else {
-            customEncoder.writeType(this, type);
         }
     }
@@ -663 +725 @@
         writeLong(duration * 1000000);
     }
+
+    /**
+     * Writes any required updates to stream
+     *
+     * @param registerUid UID of register whose entry is to be serialized after call
+     * @param entryHandle Handle of entry to be serialized after call
+     * @param handleSize Size of register's handle (in bytes)
+     * @return Whether any register updates have been written to stream
+     */
+    public boolean writeRegisterUpdates(int registerUid, int entryHandle, int handleSize) {
+        int newCounter = 0;
+        if (publishedRegisterStatus.onChangeRegisters != null) {
+            for (Register<?> register : publishedRegisterStatus.onChangeRegisters) {
+                newCounter += register.size();
+            }
+        }
+        if (newCounter < publishedRegisterStatus.publishedOnChangedEntries || entryHandle > publishedRegisterStatus.elementsWritten[registerUid]) {
+            publishedRegisterStatus.publishedOnChangedEntries = newCounter;
+
+            boolean escapeSignalWritten = false;
+
+            // Update on_change registers
+            for (int i = 0; i < SerializationInfo.MAX_PUBLISHED_REGISTERS; i++) {
+                PublishedRegisters.PerRegisterInfo localRegister = PublishedRegisters.get(i);
+                boolean may_require_update = i == registerUid || serializationTarget.getRegisterEntryEncoding(i) == SerializationInfo.RegisterEntryEncoding.PUBLISH_REGISTER_ON_CHANGE;
+                if (may_require_update && localRegister != null) {
+
+                    int currentSize = localRegister.register.size();
+                    if (currentSize > publishedRegisterStatus.elementsWritten[i]) {
+                        if (!escapeSignalWritten) {
+                            if (handleSize == 1) {
+                                writeByte(-2);
+                            } else if (handleSize == 2) {
+                                writeShort(-2);
+                            } else if (handleSize == 4) {
+                                writeInt(-2);
+                            }
+                            escapeSignalWritten = true;
+                        }
+
+                        if (getTargetInfo().getRevision() == 0) {
+                            if (publishedRegisterStatus.elementsWritten[i] == 0) {
+                                writeShort(40);
+                            }
+
+                            // compatibility with legacy parts
+                            PublishedRegisters.get(registerUid).serializeEntries(this, publishedRegisterStatus.elementsWritten[i], currentSize);
+                            writeShort(-1);
+                        } else {
+                            writeByte(i);
+                            writeInt(currentSize - publishedRegisterStatus.elementsWritten[i]);
+                            PublishedRegisters.get(registerUid).serializeEntries(this, publishedRegisterStatus.elementsWritten[i], currentSize);
+                        }
+                        publishedRegisterStatus.elementsWritten[i] = currentSize;
+                    }
+                }
+            }
+            if (escapeSignalWritten && getTargetInfo().getRevision() != 0) { // non-legacy parts require this terminator
+                writeByte(-1);
+            }
+            return escapeSignalWritten;
+        }
+        return false;
+    }
+
+    /**
+     * Writes remote register entry to stream (typically only its handle)
+     *
+     * @param entry Entry to write to stream
+     */
+    public void writeRemoteRegisterEntry(PublishedRegisters.RemoteEntryBase<?> entry) {
+        writeInt(entry.getHandle(), entry.getHandleSize());
+    }
 }

BufferInfo.java

@@ -24 +24 @@
 
 /**
+ * @author Max Reichardt
+ *
  * Buffer information
  * (can be passed to and modified by Manager (by reference))

FixedBuffer.java

@@ -63 +63 @@
     /**
      * @param capacity Capacity of buffer to allocate
-     * @param allocatDirect Allocate direct byte buffer? (in heap outside of JVM - mainly relevant for JNI code)
-     */
-    public FixedBuffer(int capacity, boolean allocatDirect) {
-        this(allocatDirect ? ByteBuffer.allocateDirect(capacity) : ByteBuffer.allocate(capacity));
+     * @param allocateDirect Allocate direct byte buffer? (in heap outside of JVM - mainly relevant for JNI code)
+     */
+    public FixedBuffer(int capacity, boolean allocateDirect) {
+        this(allocateDirect ? ByteBuffer.allocateDirect(capacity) : ByteBuffer.allocate(capacity));
     }
 

MemoryBuffer.java

@@ -39 +39 @@
  * Writing and reading concurrently is not supported - due to resize.
  */
-public class MemoryBuffer implements BinarySerializable, ConstSource, Sink, Copyable<MemoryBuffer>, GenericChangeable<MemoryBuffer> {
+public class MemoryBuffer implements BinarySerializable, Source, Sink, Copyable<MemoryBuffer>, GenericChangeable<MemoryBuffer> {
 
     /** Size of temporary array */
@@ -74 +74 @@
 
     public MemoryBuffer(int size) {
-        this(size, DEFAULT_RESIZE_FACTOR, true);
+        this(size, DEFAULT_RESIZE_FACTOR, false);
     }
 
@@ -80 +80 @@
      * @param size Initial buffer size
      * @param resizeFactor When buffer needs to be reallocated, new size is multiplied with this factor to have some bytes in reserve
-     * @param allocatDirect Allocate direct byte buffer? (in heap outside of JVM - mainly relevant for JNI code)
-     */
-    public MemoryBuffer(int size, float resizeFactor, boolean allocatDirect) {
-        backend = new FixedBuffer(size, allocatDirect);
+     * @param allocateDirect Allocate direct byte buffer? (in heap outside of JVM - mainly relevant for JNI code)
+     */
+    public MemoryBuffer(int size, float resizeFactor, boolean allocateDirect) {
+        backend = new FixedBuffer(size, allocateDirect);
         resizeReserveFactor = resizeFactor;
     }

PortDataListImpl.java

@@ -53 +53 @@
     public void serialize(BinaryOutputStream os) {
         os.writeInt(wrapped.size());
-        boolean constType = true;
+        os.writeBoolean(true); // const type?  (possibly unnecessary; if we remove it, this will break binary compatibility to 13.10 though)
         for (int i = 0; i < wrapped.size(); i++) {
-            constType &= wrapped.get(i).getClass().equals(elementType.getJavaClass());
-        }
-        os.writeBoolean(constType);
-        for (int i = 0; i < wrapped.size(); i++) {
-            if (!constType) {
-                if (wrapped.get(i) == null) {
-                    os.writeType(DataTypeBase.NULL_TYPE);
-                    continue;
-                }
-                os.writeType(DataTypeBase.findType(wrapped.get(i).getClass(), elementType));
-            }
             wrapped.get(i).serialize(os);
         }
@@ -75 +64 @@
         int size = is.readInt();
         boolean constType = is.readBoolean();
+        if (!constType) {
+            throw new RuntimeException("Non-const types are not supported");
+        }
+
         for (int i = 0; i < size; i++) {
             DataTypeBase type = elementType;
-            if (!constType) {
-                type = is.readType();
-                if (type == null || type == DataTypeBase.NULL_TYPE) {
-                    wrapped.set(i, null);
-                    continue;
-                }
-            }
             if (i < wrapped.size() && wrapped.get(i).getClass().equals(type.getJavaClass())) {
                 ((T)wrapped.get(i)).deserialize(is);

StringOutputStream.java

@@ -21 +21 @@
 //----------------------------------------------------------------------
 package org.rrlib.serialization;
+
+import org.rrlib.serialization.rtti.GenericObject;
 
 
@@ -130 +132 @@
         } else if (type.isPrimitive() || Number.class.isAssignableFrom(type) || Boolean.class.equals(type)) {
             append(object.toString());
+        } else if (type.equals(GenericObject.class)) {
+            try {
+                ((GenericObject)object).serialize(this);
+            } catch (Exception e) {
+                throw new RuntimeException("Generic object has unsupported type");
+            }
         } else {
             assert(object != null && (object.getClass() == type));

StringSerializable.java

@@ -22 +22 @@
 package org.rrlib.serialization;
 
+import org.rrlib.serialization.rtti.DataType;
+import org.rrlib.serialization.rtti.DataTypeBase;
+
 
 /**
@@ -43 +46 @@
      */
     public void deserialize(StringInputStream stream) throws Exception;
+
+
+    /** Data type of this class */
+    public final static DataTypeBase TYPE = new DataType<StringSerializable>(StringSerializable.class);
+
+    /**
+     * Empty String serializable
+     */
+    public static class Empty extends BinarySerializable.Empty implements StringSerializable {
+
+        public final static DataType<StringSerializable.Empty> TYPE = new DataType<>(StringSerializable.Empty.class, "EmptyStringSerializable");
+
+        @Override
+        public void serialize(StringOutputStream stream) {
+        }
+
+        @Override
+        public void deserialize(StringInputStream stream) throws Exception {
+        }
+
+        @Override
+        public String toString() {
+            return "";
+        }
+    }
 }

rtti/DataType.java

@@ -48 +48 @@
     public DataType(Class<?> javaClass, String name, boolean createListTypes) {
         super(name != null ? name : javaClass.getSimpleName());
-        type = Classification.PLAIN;
         this.javaClass = javaClass;
-        typeTraits = (byte)((Serialization.isBinarySerializable(javaClass) ? IS_BINARY_SERIALIZABLE : 0) |
-                            (Serialization.isStringSerializable(javaClass) ? IS_STRING_SERIALIZABLE : 0) | (Serialization.isXmlSerializable(javaClass) ? IS_XML_SERIALIZABLE : 0));
+        typeTraits = IS_DATA_TYPE |
+                     (Serialization.isBinarySerializable(javaClass) ? IS_BINARY_SERIALIZABLE : 0) |
+                     (Serialization.isStringSerializable(javaClass) ? IS_STRING_SERIALIZABLE : 0) |
+                     (Serialization.isXmlSerializable(javaClass) ? IS_XML_SERIALIZABLE : 0);
         if (javaClass.isEnum()) {
             ArrayList<String> constants = new ArrayList<String>();
@@ -62 +63 @@
 
         if (createListTypes && listType == null) {
-            listType = new DataType(this, Classification.LIST);
-            //sharedPtrListType = new DataType(this, Classification.PTR_LIST);
+            listType = new DataType(this);
         }
     }
@@ -70 +70 @@
     public DataType(Class<?> javaClass, Class<?> dedicatedListType, String name) {
         this(javaClass, name, false);
-        listType = new DataType(this, Classification.LIST);
+        listType = new DataType(this);
         listType.javaClass = dedicatedListType;
     }
@@ -77 +77 @@
      * Constructor for list types
      */
-    private DataType(DataTypeBase e, Classification type) {
-        super(type == Classification.LIST ? ("List<" + e.getName() + ">") : ("List<" + e.getName() + "*>"));
-        this.type = type;
+    private DataType(DataTypeBase e) {
+        super("List<" + e.getName() + ">");
         this.elementType = e;
-        this.typeTraits = (byte)(e.typeTraits & (IS_BINARY_SERIALIZABLE | IS_STRING_SERIALIZABLE | IS_XML_SERIALIZABLE));
+        this.typeTraits = (e.typeTraits & (IS_BINARY_SERIALIZABLE | IS_STRING_SERIALIZABLE | IS_XML_SERIALIZABLE) | IS_DATA_TYPE | IS_LIST_TYPE);
     }
 
@@ -88 +87 @@
     public Object createInstance() {
         Object result = null;
-        if (javaClass == null && (getType() == Classification.LIST || getType() == Classification.PTR_LIST)) {
+        if (javaClass == null && (typeTraits & IS_LIST_TYPE) != 0) {
             return new PortDataListImpl(getElementType());
         }

rtti/DataTypeBase.java

@@ -22 +22 @@
 package org.rrlib.serialization.rtti;
 
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -28 +27 @@
 import org.rrlib.logging.Log;
 import org.rrlib.logging.LogLevel;
+import org.rrlib.serialization.BinaryInputStream;
+import org.rrlib.serialization.BinaryOutputStream;
+import org.rrlib.serialization.PublishedRegisters;
+import org.rrlib.serialization.Register;
+import org.rrlib.serialization.PublishedRegisters.RemoteEntryBase;
 
 
@@ -47 +51 @@
 public class DataTypeBase {
 
-    /** type of data type */
-    public enum Classification {
-        PLAIN, // Plain type
-        LIST, // List of objects of same type
-        PTR_LIST, // List of objects with possibly objects of different types
-        NULL, // Null type
-        OTHER, // Other data type
-        UNKNOWN // Unknown data type in current process
-    }
-
-    /** Relevant type traits across runtime environments */
-    public static final byte IS_BINARY_SERIALIZABLE = 1 << 0;
-    public static final byte IS_STRING_SERIALIZABLE = 1 << 1;
-    public static final byte IS_XML_SERIALIZABLE = 1 << 2;
-    public static final byte IS_ENUM = 1 << 3;
+    /** Relevant type traits across runtime environments (equals C++ traits) */
+    public static final int
+    IS_BINARY_SERIALIZABLE = 1 << 8,
+    IS_STRING_SERIALIZABLE = 1 << 9,
+    IS_XML_SERIALIZABLE = 1 << 10,
+    IS_ENUM = 1 << 11,
+    IS_DATA_TYPE = 1 << 12,
+    IS_RPC_TYPE = 1 << 13,
+    // the traits below are only set in C++
+    HAS_LIST_TYPE = 1 << 14,
+    HAS_UNDERLYING_TYPE = 1 << 15,
+    IS_CAST_TO_UNDERLYING_TYPE_IMPLICIT = 1 << 16,
+    IS_REINTERPRET_CAST_FROM_UNDERLYING_TYPE_VALID = 1 << 17,
+    IS_CAST_FROM_UNDERLYING_TYPE_IMPLICIT = 1 << 18,
+    IS_UNDERLYING_TYPE_BINARY_SERIALIZATION_DIFFERENT = 1 << 19,
+    SUPPORTS_BITWISE_COPY = 1 << 20,
+    IS_INTEGRAL = 1 << 21,
+    IS_LIST_TYPE = 1 << 22,
+    HAS_TRIVIAL_DESTRUCTOR = 1 << 23;
+
+    /** Register with all registered data types */
+    private static Register<DataTypeBase> types = new Register<DataTypeBase>(32, 128, 2);
 
     /** Null type */
-    public static DataTypeBase NULL_TYPE = new DataTypeBase(true);
+    public static DataTypeBase NULL_TYPE = new DataTypeBase("NULL");
 
     /** Maximum number of annotations */
@@ -72 +83 @@
     private final static int MAX_TYPES = 2000;
 
-    /** Type of data type */
-    protected Classification type;
-
     /** Name of data type */
     private final String name;
@@ -102 +110 @@
     protected long[] enumValues;
 
-    /** List with all registered data types (preallocated to avoid reallocations => concurrent use is possible) */
-    private static ArrayList<DataTypeBase> types = new ArrayList<DataTypeBase>(MAX_TYPES);
-
     /** Lookup for data type annotation index */
     private static final HashMap < Class<?>, Integer > annotationIndexLookup = new HashMap < Class<?>, Integer > ();
@@ -112 +117 @@
 
     /** Type traits of this type (bit vector - see constants above) */
-    protected byte typeTraits;
+    protected int typeTraits;
 
     ///** Is this a remote type? */
@@ -121 +126 @@
             uid = (short)types.size();
             this.name = name;
-            for (DataTypeBase type : types) {
-                if (type.getName().equals(name)) {
+            for (int i = 0, n = types.size(); i < n; i++) {
+                if (types.get(i).getName().equals(name)) {
                     Log.log(LogLevel.WARNING, "Two types with the same name were registered: " + name);
                 }
@@ -136 +141 @@
     }
 
-    private DataTypeBase(boolean nulltype) {
-        this.name = "NULL";
-        this.uid = -1;
-        this.type = Classification.NULL;
-    }
-
     /**
      * @return Name of data type
@@ -164 +163 @@
 
     /**
-     * @return return "Type" of data type (see enum)
-     */
-    public Classification getType() {
-        return type;
-    }
-
-    /**
      * @return In case of element: list type (std::vector<T>)
      */
@@ -201 +193 @@
      * @return Bit vector of type traits
      */
-    public byte getTypeTraits() {
+    public int getTypeTraits() {
         return typeTraits;
     }
@@ -322 +314 @@
     /**
      * Can object of this data type be converted to specified type?
-     * (In C++ currently only returns true, when types are equal)
      *
      * @param dataType Other type
@@ -332 +323 @@
             return true;
         }
-        if (type == Classification.NULL || dataType.type == Classification.NULL) {
-            return false;
-        }
-        if (getType() == Classification.UNKNOWN || dataType.getType() == Classification.UNKNOWN) {
-            return false;
-        }
-        if (getType() == Classification.LIST && dataType.getType() == Classification.LIST) {
-            return getElementType().isConvertibleTo(dataType.getElementType());
+        if ((typeTraits & dataType.typeTraits & IS_LIST_TYPE) != 0) {
+            return getElementType() != null && dataType.getElementType() != null && getElementType().isConvertibleTo(dataType.getElementType());
         }
         if ((javaClass != null) && (dataType.javaClass != null)) {
@@ -392 +377 @@
         return enumConstants;
     }
+
     /**
      * @return Array with enum values - if this is a remote enum type with non-standard values
@@ -398 +384 @@
         return enumValues;
     }
+
+    /**
+     * @param stream Stream to serialize this type to
+     */
+    public void serialize(BinaryOutputStream stream) {
+        if (types.writeEntry(stream, getUid())) {
+            stream.writeString(getName());
+        }
+    }
+
+    /**
+     * @param stream Stream to deserialize (local) type from
+     * @return Deserialized type
+     */
+    public static DataTypeBase deserialize(BinaryInputStream stream) throws Exception {
+        DataTypeBase result = types.readEntry(stream);
+        if (result == null) {
+            return findType(stream.readString());
+        }
+        return result;
+    }
+
+    /**
+     * Registers type register for use in auto-publishing mechanism.
+     *
+     * @param uid Uid to assign to register (must be <= MAX_PUBLISHED_REGISTERS)
+     * @param remoteEntryClass Type to deserialize in remote runtime environment. It needs to be derived from PublishedRegisters.RemoteEntryBase and be default-constructible.
+     * @throw Throws exception on already occupied uid
+     */
+    public static void registerForPublishing(int uid, Class<? extends RemoteEntryBase<?>> remoteEntryClass) throws Exception {
+        PublishedRegisters.register(types, uid, remoteEntryClass);
+    }
 }