Skip to content

re-added option to disable UTF-8 encoding for metadata #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
twincitiesguy merged 3 commits into from
May 5, 2017
Merged

re-added option to disable UTF-8 encoding for metadata #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
786d7ea
re-added option to disable UTF-8 encoding for metadata
May 4, 2017
72d23fa
fixed potential NPE in RestUtil.parseMetadataHeader
May 5, 2017
ce13f58
added test case for mixing UTF-8 encoding/not-encoding
May 5, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions settings.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
rootProject.name = 'atmos-client'

2 changes: 1 addition & 1 deletion src/main/java/com/emc/atmos/api/AbstractAtmosApi.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ public void deleteAccessToken( URL url ) {
@Override
public PreSignedRequest preSignRequest( Request request, Date expiration ) throws MalformedURLException {
URI uri = config.resolvePath( request.getServiceRelativePath(), request.getQuery() );
Map<String, List<Object>> headers = request.generateHeaders();
Map<String, List<Object>> headers = request.generateHeaders( config.isEncodeUtf8() );

String contentType = null;
if ( request instanceof ContentRequest ) contentType = ((ContentRequest) request).getContentType();
Expand Down
16 changes: 16 additions & 0 deletions src/main/java/com/emc/atmos/api/AtmosConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ public class AtmosConfig extends AbstractConfig {
private URI proxyUri;
private String proxyUser;
private String proxyPassword;
private boolean encodeUtf8 = true;

/**
* Creates a new instance with default parameters. tokenId, secretKey and at least one endpoint must be provided
Expand Down Expand Up @@ -242,4 +243,19 @@ public String getProxyPassword() {
public void setProxyPassword( String proxyPassword ) {
this.proxyPassword = proxyPassword;
}

/**
* Gets whether UTF-8 encoding is enabled
*/
public boolean isEncodeUtf8() {
return encodeUtf8;
}

/**
* Sets whether to encode metadata and keys using HTTP UTF-8 escape sequences to allow extended character use.
* Default is true.
*/
public void setEncodeUtf8(boolean encodeUtf8) {
this.encodeUtf8 = encodeUtf8;
}
}
8 changes: 5 additions & 3 deletions src/main/java/com/emc/atmos/api/RestUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,14 +219,16 @@ public static ObjectId parseObjectId( String path ) {
throw new AtmosException( "Cannot find object ID in path" + path );
}

public static Map<String, Metadata> parseMetadataHeader( String headerValue, boolean listable ) {
public static Map<String, Metadata> parseMetadataHeader( String headerValue, boolean listable, boolean decodeUtf8 ) {
Map<String, Metadata> metadataMap = new TreeMap<String, Metadata>();
if ( headerValue == null ) return metadataMap;
String[] pairs = headerValue.split( ",(?=[^,]+=)" ); // comma with key as look-ahead (not part of match)
for ( String pair : pairs ) {
String[] components = pair.split( "=", 2 );
String name = HttpUtil.decodeUtf8( components[0].trim() );
String value = components.length > 1 ? HttpUtil.decodeUtf8( components[1] ) : null;
String name = components[0].trim();
if ( decodeUtf8 ) name = HttpUtil.decodeUtf8( name );
String value = components.length > 1 ? components[1] : null;
if ( value != null && decodeUtf8 ) value = HttpUtil.decodeUtf8( value );
Metadata metadata = new Metadata( name, value, listable );
metadataMap.put( name, metadata );
}
Expand Down
12 changes: 9 additions & 3 deletions src/main/java/com/emc/atmos/api/bean/ReadObjectResponse.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,14 +53,20 @@ public void setObject( T object ) {
this.object = object;
}

public synchronized ObjectMetadata getMetadata() {
public ObjectMetadata getMetadata() {
return getMetadata( true );
}

public synchronized ObjectMetadata getMetadata( boolean decodeUtf8 ) {
if ( metadata == null ) {
Acl acl = new Acl( RestUtil.parseAclHeader( getFirstHeader( RestUtil.XHEADER_USER_ACL ) ),
RestUtil.parseAclHeader( getFirstHeader( RestUtil.XHEADER_GROUP_ACL ) ) );

Map<String, Metadata> metaMap = new TreeMap<String, Metadata>();
metaMap.putAll( RestUtil.parseMetadataHeader( getFirstHeader( RestUtil.XHEADER_META ), false ) );
metaMap.putAll( RestUtil.parseMetadataHeader( getFirstHeader( RestUtil.XHEADER_LISTABLE_META ), true ) );
metaMap.putAll( RestUtil.parseMetadataHeader( getFirstHeader( RestUtil.XHEADER_META ),
false, decodeUtf8 ) );
metaMap.putAll( RestUtil.parseMetadataHeader( getFirstHeader( RestUtil.XHEADER_LISTABLE_META ),
true, decodeUtf8 ) );

String wsChecksumHeader = getFirstHeader( RestUtil.XHEADER_WSCHECKSUM );
ChecksumValue wsChecksum = wsChecksumHeader == null ? null : new ChecksumValueImpl( wsChecksumHeader );
Expand Down
73 changes: 48 additions & 25 deletions src/main/java/com/emc/atmos/api/jersey/AtmosApiClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -288,8 +288,10 @@ public ListDirectoryResponse listDirectory( ListDirectoryRequest request ) {
public void move( ObjectPath oldPath, ObjectPath newPath, boolean overwrite ) {
WebResource resource = client.resource( config.resolvePath( oldPath.getRelativeResourcePath(), "rename" ) );
WebResource.Builder builder = resource.getRequestBuilder();
builder.header( RestUtil.XHEADER_UTF8, "true" )
.header( RestUtil.XHEADER_PATH, HttpUtil.encodeUtf8( newPath.getPath() ) );
builder.header( RestUtil.XHEADER_PATH,
config.isEncodeUtf8() ? HttpUtil.encodeUtf8( newPath.getPath() ) : newPath.getPath() );
if ( config.isEncodeUtf8() ) builder.header( RestUtil.XHEADER_UTF8, "true" );

if ( overwrite ) builder.header( RestUtil.XHEADER_FORCE, "true" );

// workaround for clients that set a default content-type for POSTs
Expand All @@ -305,20 +307,22 @@ public Map<String, Boolean> getUserMetadataNames( ObjectIdentifier identifier )
if ( identifier instanceof ObjectKey )
builder.header( RestUtil.XHEADER_POOL, ((ObjectKey) identifier).getBucket() );

ClientResponse response = builder.header( RestUtil.XHEADER_UTF8, "true" ).get( ClientResponse.class );
if ( config.isEncodeUtf8() ) builder.header( RestUtil.XHEADER_UTF8, "true" );

ClientResponse response = builder.get( ClientResponse.class );

Map<String, Boolean> metaNames = new TreeMap<String, Boolean>();

String nameString = response.getHeaders().getFirst( RestUtil.XHEADER_TAGS );
if ( nameString != null ) {
for ( String name : nameString.split( "," ) )
metaNames.put( HttpUtil.decodeUtf8( name.trim() ), false );
metaNames.put( config.isEncodeUtf8() ? HttpUtil.decodeUtf8( name.trim() ) : name.trim(), false );
}

nameString = response.getHeaders().getFirst( RestUtil.XHEADER_LISTABLE_TAGS );
if ( nameString != null ) {
for ( String name : nameString.split( "," ) )
metaNames.put( HttpUtil.decodeUtf8( name.trim() ), true );
metaNames.put( config.isEncodeUtf8() ? HttpUtil.decodeUtf8( name.trim() ) : name.trim(), true );
}

response.close();
Expand All @@ -336,17 +340,20 @@ public Map<String, Metadata> getUserMetadata( ObjectIdentifier identifier, Strin

if ( metadataNames != null ) {
for ( String name : metadataNames ) {
builder.header( RestUtil.XHEADER_TAGS, HttpUtil.encodeUtf8( name ) );
if ( config.isEncodeUtf8() ) name = HttpUtil.encodeUtf8( name );
builder.header( RestUtil.XHEADER_TAGS, name );
}
}

ClientResponse response = builder.header( RestUtil.XHEADER_UTF8, "true" ).get( ClientResponse.class );
if ( config.isEncodeUtf8() ) builder.header( RestUtil.XHEADER_UTF8, "true" );

ClientResponse response = builder.get( ClientResponse.class );

Map<String, Metadata> metaMap = new TreeMap<String, Metadata>();
metaMap.putAll( RestUtil.parseMetadataHeader( response.getHeaders().getFirst( RestUtil.XHEADER_META ),
false ) );
false, config.isEncodeUtf8() ) );
metaMap.putAll( RestUtil.parseMetadataHeader( response.getHeaders().getFirst( RestUtil.XHEADER_LISTABLE_META ),
true ) );
true, config.isEncodeUtf8() ) );

response.close();

Expand All @@ -363,15 +370,19 @@ public Map<String, Metadata> getSystemMetadata( ObjectIdentifier identifier, Str

if ( metadataNames != null ) {
for ( String name : metadataNames ) {
builder.header( RestUtil.XHEADER_TAGS, HttpUtil.encodeUtf8( name ) );
if ( config.isEncodeUtf8() ) name = HttpUtil.encodeUtf8( name );
builder.header( RestUtil.XHEADER_TAGS, name );
}
}

ClientResponse response = builder.header( RestUtil.XHEADER_UTF8, "true" ).get( ClientResponse.class );
if ( config.isEncodeUtf8() ) builder.header( RestUtil.XHEADER_UTF8, "true" );

ClientResponse response = builder.get( ClientResponse.class );

response.close();

return RestUtil.parseMetadataHeader( response.getHeaders().getFirst( RestUtil.XHEADER_META ), false );
return RestUtil.parseMetadataHeader( response.getHeaders().getFirst( RestUtil.XHEADER_META ),
false, config.isEncodeUtf8() );
}

@Override
Expand All @@ -393,18 +404,20 @@ public ObjectMetadata getObjectMetadata( ObjectIdentifier identifier ) {
if ( identifier instanceof ObjectKey )
builder.header( RestUtil.XHEADER_POOL, ((ObjectKey) identifier).getBucket() );

ClientResponse response = builder.header( RestUtil.XHEADER_UTF8, "true" ).head();
if ( config.isEncodeUtf8() ) builder.header( RestUtil.XHEADER_UTF8, "true" );

ClientResponse response = builder.head();

Acl acl = new Acl( RestUtil.parseAclHeader( response.getHeaders().getFirst( RestUtil.XHEADER_USER_ACL ) ),
RestUtil.parseAclHeader( response.getHeaders()
.getFirst( RestUtil.XHEADER_GROUP_ACL ) ) );

Map<String, Metadata> metaMap = new TreeMap<String, Metadata>();
metaMap.putAll( RestUtil.parseMetadataHeader( response.getHeaders().getFirst( RestUtil.XHEADER_META ),
false ) );
false, config.isEncodeUtf8() ) );
metaMap.putAll( RestUtil.parseMetadataHeader( response.getHeaders()
.getFirst( RestUtil.XHEADER_LISTABLE_META ),
true ) );
true, config.isEncodeUtf8() ) );

String wsChecksumHeader = response.getHeaders().getFirst( RestUtil.XHEADER_WSCHECKSUM );
ChecksumValue wsChecksum = wsChecksumHeader == null ? null : new ChecksumValueImpl( wsChecksumHeader );
Expand All @@ -429,14 +442,19 @@ public void setUserMetadata( ObjectIdentifier identifier, Metadata... metadata )
builder.header( RestUtil.XHEADER_POOL, ((ObjectKey) identifier).getBucket() );

for ( Metadata oneMetadata : metadata ) {
if ( oneMetadata.isListable() )
builder.header( RestUtil.XHEADER_LISTABLE_META, oneMetadata.toASCIIString() );
else builder.header( RestUtil.XHEADER_META, oneMetadata.toASCIIString() );
if ( oneMetadata.isListable() ) {
builder.header( RestUtil.XHEADER_LISTABLE_META,
config.isEncodeUtf8() ? oneMetadata.toASCIIString() : oneMetadata.toString() );
} else {
builder.header( RestUtil.XHEADER_META,
config.isEncodeUtf8() ? oneMetadata.toASCIIString() : oneMetadata.toString() );
}
}

// workaround for clients that set a default content-type for POSTs
builder.type( RestUtil.TYPE_DEFAULT );
builder.header( RestUtil.XHEADER_UTF8, "true" ).post();
if ( config.isEncodeUtf8() ) builder.header( RestUtil.XHEADER_UTF8, "true" );
builder.post();
}

@Override
Expand All @@ -448,10 +466,13 @@ public void deleteUserMetadata( ObjectIdentifier identifier, String... names ) {
builder.header( RestUtil.XHEADER_POOL, ((ObjectKey) identifier).getBucket() );

for ( String name : names ) {
builder.header( RestUtil.XHEADER_TAGS, HttpUtil.encodeUtf8( name ) );
if ( config.isEncodeUtf8() ) name = HttpUtil.encodeUtf8( name );
builder.header( RestUtil.XHEADER_TAGS, name );
}

builder.header( RestUtil.XHEADER_UTF8, "true" ).delete();
if ( config.isEncodeUtf8() ) builder.header( RestUtil.XHEADER_UTF8, "true" );

builder.delete();
}

@Override
Expand All @@ -460,15 +481,17 @@ public Set<String> listMetadata( String metadataName ) {
WebResource.Builder builder = client.resource( uri ).getRequestBuilder();

if ( metadataName != null )
builder.header( RestUtil.XHEADER_TAGS, HttpUtil.encodeUtf8( metadataName ) );
builder.header( RestUtil.XHEADER_TAGS,
config.isEncodeUtf8() ? HttpUtil.encodeUtf8( metadataName ) : metadataName );
if ( config.isEncodeUtf8() ) builder.header( RestUtil.XHEADER_UTF8, "true" );

ClientResponse response = builder.header( RestUtil.XHEADER_UTF8, "true" ).get( ClientResponse.class );
ClientResponse response = builder.get( ClientResponse.class );
String headerValue = response.getHeaders().getFirst( RestUtil.XHEADER_LISTABLE_TAGS );

Set<String> names = new TreeSet<String>();
if ( headerValue == null ) return names;
for ( String name : headerValue.split( "," ) )
names.add( HttpUtil.decodeUtf8( name.trim() ) );
names.add( config.isEncodeUtf8() ? HttpUtil.decodeUtf8( name.trim() ) : name.trim() );

response.close();

Expand Down Expand Up @@ -679,7 +702,7 @@ protected WebResource.Builder build( Request request ) {
builder.type( RestUtil.TYPE_DEFAULT );
}

return addHeaders( builder, request.generateHeaders() );
return addHeaders( builder, request.generateHeaders( config.isEncodeUtf8() ) );
}

protected WebResource.Builder addHeaders( WebResource.Builder builder, Map<String, List<Object>> headers ) {
Expand Down
10 changes: 5 additions & 5 deletions src/main/java/com/emc/atmos/api/request/CreateAccessTokenRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@

import java.util.List;
import java.util.Map;
import java.util.TreeMap;

/**
* Represents a request to create an anonymous access token.
Expand All @@ -55,17 +54,18 @@ public String getMethod() {
}

@Override
public Map<String, List<Object>> generateHeaders() {
Map<String, List<Object>> headers = super.generateHeaders();
public Map<String, List<Object>> generateHeaders( boolean encodeUtf8 ) {
Map<String, List<Object>> headers = super.generateHeaders( encodeUtf8 );

// target object
if ( identifier != null ) {
if ( identifier instanceof ObjectId )
RestUtil.addValue( headers, RestUtil.XHEADER_OBJECTID, identifier );
else if ( identifier instanceof ObjectPath ) {
// enable UTF-8
RestUtil.addValue( headers, RestUtil.XHEADER_UTF8, "true" );
RestUtil.addValue( headers, RestUtil.XHEADER_PATH, HttpUtil.encodeUtf8( identifier.toString() ) );
if ( encodeUtf8 ) RestUtil.addValue( headers, RestUtil.XHEADER_UTF8, "true" );
RestUtil.addValue( headers, RestUtil.XHEADER_PATH,
encodeUtf8 ? HttpUtil.encodeUtf8( identifier.toString() ) : identifier.toString() );
} else
throw new UnsupportedOperationException(
"Only object ID and path are currently supported in access tokens" );
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/com/emc/atmos/api/request/CreateObjectRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@ public String getServiceRelativePath() {
}

@Override
public Map<String, List<Object>> generateHeaders() {
Map<String, List<Object>> headers = super.generateHeaders();
public Map<String, List<Object>> generateHeaders( boolean encodeUtf8 ) {
Map<String, List<Object>> headers = super.generateHeaders( encodeUtf8 );

// custom object ID
if ( getCustomObjectId() != null ) RestUtil.addValue( headers, RestUtil.XHEADER_OBJECT_ID, customObjectId );
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/emc/atmos/api/request/CreateSubtenantRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ public String getMethod() {
}

@Override
public Map<String, List<Object>> generateHeaders() {
public Map<String, List<Object>> generateHeaders( boolean encodeUtf8 ) {
Map<String, List<Object>> headers = new HashMap<String, List<Object>>();
if(objectVirtualPoolId != null) {
headers.put(RestUtil.XHEADER_OBJECT_VPOOL,
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/com/emc/atmos/api/request/ListMetadataRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@ public abstract class ListMetadataRequest<T extends ListMetadataRequest<T>> exte
protected boolean includeMetadata;

@Override
public Map<String, List<Object>> generateHeaders() {
Map<String, List<Object>> headers = super.generateHeaders();
public Map<String, List<Object>> generateHeaders( boolean encodeUtf8 ) {
Map<String, List<Object>> headers = super.generateHeaders( encodeUtf8 );

if ( includeMetadata ) {
RestUtil.addValue( headers, RestUtil.XHEADER_INCLUDE_META, 1 );
Expand Down
7 changes: 4 additions & 3 deletions src/main/java/com/emc/atmos/api/request/ListObjectsRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,11 @@ public String getMethod() {
}

@Override
public Map<String, List<Object>> generateHeaders() {
Map<String, List<Object>> headers = super.generateHeaders();
public Map<String, List<Object>> generateHeaders( boolean encodeUtf8 ) {
Map<String, List<Object>> headers = super.generateHeaders( encodeUtf8 );

RestUtil.addValue( headers, RestUtil.XHEADER_TAGS, HttpUtil.encodeUtf8( metadataName ) );
RestUtil.addValue( headers, RestUtil.XHEADER_TAGS,
encodeUtf8 ? HttpUtil.encodeUtf8( metadataName ) : metadataName );

return headers;
}
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/com/emc/atmos/api/request/ListRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,10 @@ public abstract class ListRequest<T extends ListRequest<T>> extends Request {
protected abstract T me();

@Override
public Map<String, List<Object>> generateHeaders() {
public Map<String, List<Object>> generateHeaders( boolean encodeUtf8 ) {
Map<String, List<Object>> headers = new TreeMap<String, List<Object>>();

RestUtil.addValue( headers, RestUtil.XHEADER_UTF8, "true" );
if ( encodeUtf8 ) RestUtil.addValue( headers, RestUtil.XHEADER_UTF8, "true" );

if ( limit > 0 ) RestUtil.addValue( headers, RestUtil.XHEADER_LIMIT, limit );

Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/emc/atmos/api/request/ObjectRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public abstract class ObjectRequest<T extends ObjectRequest<T>> extends Request
protected abstract T me();

@Override
public Map<String, List<Object>> generateHeaders() {
public Map<String, List<Object>> generateHeaders( boolean encodeUtf8 ) {
Map<String, List<Object>> headers = new TreeMap<String, List<Object>>();

if ( identifier != null && identifier instanceof ObjectKey ) {
Expand Down
13 changes: 8 additions & 5 deletions src/main/java/com/emc/atmos/api/request/PutObjectRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,18 +55,21 @@ public PutObjectRequest() {
}

@Override
public Map<String, List<Object>> generateHeaders() {
Map<String, List<Object>> headers = super.generateHeaders();
public Map<String, List<Object>> generateHeaders( boolean encodeUtf8 ) {
Map<String, List<Object>> headers = super.generateHeaders( encodeUtf8 );

// enable UTF-8
if ( !getUserMetadata().isEmpty() ) RestUtil.addValue( headers, RestUtil.XHEADER_UTF8, "true" );
if ( !getUserMetadata().isEmpty() && encodeUtf8 )
RestUtil.addValue( headers, RestUtil.XHEADER_UTF8, "true" );

// metadata
for ( Metadata metadata : getUserMetadata() ) {
if ( metadata.isListable() )
RestUtil.addValue( headers, RestUtil.XHEADER_LISTABLE_META, metadata.toASCIIString() );
RestUtil.addValue( headers, RestUtil.XHEADER_LISTABLE_META,
encodeUtf8 ? metadata.toASCIIString() : metadata.toString() );
else
RestUtil.addValue( headers, RestUtil.XHEADER_META, metadata.toASCIIString() );
RestUtil.addValue( headers, RestUtil.XHEADER_META,
encodeUtf8 ? metadata.toASCIIString() : metadata.toString() );
}

// acl
Expand Down
Loading