Skip to content

Step-parser: Allow for larger files #32

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
HakonEriksen wants to merge 1 commit into
base: 1.2-stable
Choose a base branch
from
Open

Step-parser: Allow for larger files #32

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions Step/src/org/bimserver/step/ByteBuffer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,40 @@
package org.bimserver.step;

import java.io.IOException;
import java.io.InputStream;

class ByteBuffer {

private ByteBufferPage first = new ByteBufferPage(1 << 20);
private ByteBufferPage writePage = first;
private ByteBufferPage readPage = first;
private int writePageIndex = 0;
private int readPageIndex = 0;
private long length;

public ByteBuffer() {
}

public ByteBuffer(byte[] content) {
append(content, 0, content.length);
}

public ByteBuffer(String content) {
this(content.getBytes());
}

public ByteBuffer(InputStream in) throws IOException {
append(in);
}

public void append(InputStream in) throws IOException {
byte[] buffer = new byte[4096];
int length = 0;
while ((length = in.read(buffer, 0, buffer.length)) > 0) {
append(buffer, 0, length);
}
}

public void append(byte[] values, int offset, int length) {
int pageIndex = (int) (this.length >> 20);
ByteBufferPage page = writePage;
Expand Down
8 changes: 6 additions & 2 deletions Step/src/org/bimserver/step/StepAttribute.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,12 @@ public interface StepAttribute {
String getTokenValue();
boolean isUnset();
boolean isRedeclared();
boolean isBinary();
boolean isEnum();
boolean isList();
boolean isInstanceName();
boolean isInline();
boolean isInstanceName();
boolean isList();
boolean isInteger();
boolean isReal();
boolean isString();
}
176 changes: 103 additions & 73 deletions Step/src/org/bimserver/step/StepAttributeImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ public class StepAttributeImpl implements StepAttribute {
private final ByteBuffer dataBuffer;
private final TokenBuffer tokenBuffer;
private final int index;
private long token;
private StepToken token;

private static byte[] buffer = new byte[0];

Expand All @@ -23,81 +23,92 @@ public int tokenLength() {

@Override
public Object getValue() {
long offset = StepTokenizer.tokenPosition(token);
int length = StepTokenizer.tokenLength(token);
switch (StepTokenizer.tokenType(token)) {
case StepTokenizer.TOKEN_INTEGER:
{
if (buffer.length < length) {
buffer = new byte[length];
long offset = token.getPosition();
int length = token.getLength();
switch (token.getType()) {
case StepToken.TOKEN_INTEGER: {
if (buffer.length < length) {
buffer = new byte[length];
}
dataBuffer.bytesAt(buffer, offset, length);
String str = new String(buffer, 0, length);
try {
return Long.valueOf(str);
} catch (NumberFormatException e) {
}
throw new RuntimeException("Invalid integer " + new String(buffer));
}
dataBuffer.bytesAt(buffer, offset, length);
String str = new String(buffer, 0, length);
try {
return Long.valueOf(str);
} catch (NumberFormatException e) {
case StepToken.TOKEN_REAL: {
if (buffer.length < length) {
buffer = new byte[length];
}
dataBuffer.bytesAt(buffer, offset, length);
String str = new String(buffer, 0, length);
try {
return Double.valueOf(str);
} catch (NumberFormatException e) {
}
throw new RuntimeException("Invalid real " + new String(buffer));
}
throw new RuntimeException("Invalid integer " + new String(buffer));
}
case StepTokenizer.TOKEN_REAL:
{
if (buffer.length < length) {
buffer = new byte[length];
}
dataBuffer.bytesAt(buffer, offset, length);
String str = new String(buffer, 0, length);
try {
return Double.valueOf(str);
} catch (NumberFormatException e) {
}
throw new RuntimeException("Invalid real " + new String(buffer));
}
case StepTokenizer.TOKEN_INSTANCE_NAME:
{
if (buffer.length < length) {
buffer = new byte[length];
}
dataBuffer.bytesAt(buffer, offset, length);
String str = new String(buffer, 0, length);
try {
return Long.valueOf(str);
} catch (NumberFormatException e) {
case StepToken.TOKEN_INSTANCE_NAME: {
offset++;
length--;
if (buffer.length < length) {
buffer = new byte[length];
}
dataBuffer.bytesAt(buffer, offset, length);
String str = new String(buffer, 0, length);
try {
return Long.valueOf(str);
} catch (NumberFormatException e) {
}
throw new RuntimeException("Invalid instance id " + new String(buffer));
}
throw new RuntimeException("Invalid instance id " + new String(buffer));
}
case StepTokenizer.TOKEN_ENUM:
{
if (buffer.length < length) {
buffer = new byte[length];
case StepToken.TOKEN_ENUM: {
offset++;
length -= 2;
if (buffer.length < length) {
buffer = new byte[length];
}
dataBuffer.bytesAt(buffer, offset, length);
return new String(buffer, 0, length);
}
dataBuffer.bytesAt(buffer, offset, length);
return new String(buffer, 0, length);
}
case StepTokenizer.TOKEN_STRING:
{
if (buffer.length < length) {
buffer = new byte[length];
case StepToken.TOKEN_BINARY: {
offset++;
length -= 2;
if (buffer.length < length) {
buffer = new byte[length];
}
dataBuffer.bytesAt(buffer, offset, length);
return new String(buffer, 0, length);
}
dataBuffer.bytesAt(buffer, offset, length);
return StepStringDecoder.decode(buffer, 0, length);
}
case StepTokenizer.TOKEN_IDENTIFIER:
if (StepTokenizer.tokenType(tokenBuffer.tokenAt(index + 1)) == StepTokenizer.TOKEN_LPAREN) {
return new StepEntityInstanceImpl(dataBuffer, tokenBuffer, index);
} else {
throw new RuntimeException("Missing attribute list for inline instance");
case StepToken.TOKEN_STRING: {
offset++;
length -= 2;
if (buffer.length < length) {
buffer = new byte[length];
}
dataBuffer.bytesAt(buffer, offset, length);
return StepStringDecoder.decode(buffer, 0, length);
}
case StepTokenizer.TOKEN_UNSET:
case StepTokenizer.TOKEN_REDECLARED:
return null;
default:
throw new RuntimeException("Unknown token");
case StepToken.TOKEN_IDENTIFIER:
if (tokenBuffer.tokenAt(index + 1).getType() == StepToken.TOKEN_LPAREN) {
return new StepEntityInstanceImpl(dataBuffer, tokenBuffer, index);
} else {
throw new RuntimeException("Missing attribute list for inline instance");
}
case StepToken.TOKEN_UNSET:
case StepToken.TOKEN_REDECLARED:
return null;
default:
throw new RuntimeException("Unknown token");
}
}

@Override
public String getTokenValue() {
long offset = StepTokenizer.tokenPosition(token);
int length = StepTokenizer.tokenLength(token);
long offset = token.getPosition();
int length = token.getLength();
if (buffer.length < length) {
buffer = new byte[length];
}
Expand All @@ -107,17 +118,17 @@ public String getTokenValue() {

@Override
public boolean isUnset() {
return StepTokenizer.tokenType(token) == StepTokenizer.TOKEN_UNSET;
return token.getType() == StepToken.TOKEN_UNSET;
}

public void markAsUnset() {
token = StepTokenizer.token(StepTokenizer.tokenType(StepTokenizer.TOKEN_UNSET), StepTokenizer.tokenPosition(token), StepTokenizer.tokenLength(token));
token = new StepToken(StepToken.TOKEN_UNSET, token.getPosition(), token.getLength());
tokenBuffer.set(index, token);
}

@Override
public boolean isRedeclared() {
return StepTokenizer.tokenType(token) == StepTokenizer.TOKEN_REDECLARED;
return token.getType() == StepToken.TOKEN_REDECLARED;
}

@Override
Expand All @@ -127,18 +138,37 @@ public boolean isList() {

@Override
public boolean isInstanceName() {
return StepTokenizer.tokenType(token) == StepTokenizer.TOKEN_INSTANCE_NAME;
return token.getType() == StepToken.TOKEN_INSTANCE_NAME;
}

@Override
public boolean isEnum() {
return StepTokenizer.tokenType(token) == StepTokenizer.TOKEN_ENUM;
return token.getType() == StepToken.TOKEN_ENUM;
}

@Override
public boolean isInline() {
return StepTokenizer.tokenType(token) == StepTokenizer.TOKEN_IDENTIFIER &&
StepTokenizer.tokenType(tokenBuffer.tokenAt(index + 1)) == StepTokenizer.TOKEN_LPAREN;
return token.getType() == StepToken.TOKEN_IDENTIFIER &&
tokenBuffer.tokenAt(index + 1).getType() == StepToken.TOKEN_LPAREN;
}

@Override
public boolean isString() {
return token.getType() == StepToken.TOKEN_STRING;
}

@Override
public boolean isInteger() {
return token.getType() == StepToken.TOKEN_INTEGER;
}

@Override
public boolean isReal() {
return token.getType() == StepToken.TOKEN_REAL;
}

@Override
public boolean isBinary() {
return token.getType() == StepToken.TOKEN_BINARY;
}
}
60 changes: 30 additions & 30 deletions Step/src/org/bimserver/step/StepAttributeIterator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,40 +17,41 @@ public StepAttributeIterator(ByteBuffer dataBuffer,

@Override
public boolean hasNext() {
int tokenType = StepTokenizer.tokenType(tokenBuffer.tokenAt(index));
return tokenType != StepTokenizer.TOKEN_RPAREN;
int tokenType = tokenBuffer.tokenAt(index).getType();
return tokenType != StepToken.TOKEN_RPAREN;
}

@Override
public StepAttribute next() {
long token = tokenBuffer.tokenAt(index);
StepToken token = tokenBuffer.tokenAt(index);
StepAttribute attribute = null;
switch (StepTokenizer.tokenType(token)) {
case StepTokenizer.TOKEN_INTEGER:
case StepTokenizer.TOKEN_REAL:
case StepTokenizer.TOKEN_STRING:
case StepTokenizer.TOKEN_UNSET:
case StepTokenizer.TOKEN_REDECLARED:
case StepTokenizer.TOKEN_INSTANCE_NAME:
case StepTokenizer.TOKEN_ENUM:
attribute = new StepAttributeImpl(dataBuffer, tokenBuffer, index);
break;
case StepTokenizer.TOKEN_IDENTIFIER:
attribute = new StepAttributeImpl(dataBuffer, tokenBuffer, index);
if (StepTokenizer.tokenType(tokenBuffer.tokenAt(index + 1)) == StepTokenizer.TOKEN_LPAREN) {
int listLength = new StepAttributeListImpl(dataBuffer,
tokenBuffer, index + 1).tokenLength();
index += listLength;
}
break;
case StepTokenizer.TOKEN_LPAREN:
StepAttributeListImpl list = new StepAttributeListImpl(
dataBuffer, tokenBuffer, index);
index += list.tokenLength() - 1;
attribute = list;
break;
default:
throw new RuntimeException();
switch (token.getType()) {
case StepToken.TOKEN_INTEGER:
case StepToken.TOKEN_REAL:
case StepToken.TOKEN_STRING:
case StepToken.TOKEN_UNSET:
case StepToken.TOKEN_REDECLARED:
case StepToken.TOKEN_INSTANCE_NAME:
case StepToken.TOKEN_ENUM:
case StepToken.TOKEN_BINARY:
attribute = new StepAttributeImpl(dataBuffer, tokenBuffer, index);
break;
case StepToken.TOKEN_IDENTIFIER:
attribute = new StepAttributeImpl(dataBuffer, tokenBuffer, index);
if (tokenBuffer.tokenAt(index + 1).getType() == StepToken.TOKEN_LPAREN) {
int listLength = new StepAttributeListImpl(dataBuffer,
tokenBuffer, index + 1).tokenLength();
index += listLength;
}
break;
case StepToken.TOKEN_LPAREN:
StepAttributeListImpl list = new StepAttributeListImpl(
dataBuffer, tokenBuffer, index);
index += list.tokenLength() - 1;
attribute = list;
break;
default:
throw new RuntimeException();
}
index++;
return attribute;
Expand All @@ -60,5 +61,4 @@ public StepAttribute next() {
public void remove() {
}


}
Loading