diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..55006a6 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,38 @@ +name: CI + +on: + push: + branches: + - "**" + pull_request: + branches: + - "**" + +jobs: + test: + name: Test and Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK 1.8 + uses: actions/setup-java@v4 + with: + java-version: '8' + distribution: 'temurin' + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v3 + + - name: Run Spotless Check + run: ./gradlew spotlessCheck + + - name: Run Tests + run: ./gradlew test + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results + path: build/reports/tests/test/ diff --git a/.github/workflows/gem-push.yml b/.github/workflows/gem-push.yml index 290aa77..f4f3ff2 100644 --- a/.github/workflows/gem-push.yml +++ b/.github/workflows/gem-push.yml @@ -3,14 +3,8 @@ name: Ruby Gem on: workflow_dispatch: push: - branches: - - "master" tags: - - "*" - pull_request: - branches: - - "master" - types: [opened, synchronize] + - "v*" jobs: build: @@ -20,14 +14,15 @@ jobs: packages: write contents: read steps: - - uses: actions/checkout@v1 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 + - uses: actions/checkout@v4 + - name: Set up Java8 + uses: actions/setup-java@v4 with: - java-version: 1.8 + distribution: "temurin" + java-version: "8" - name: push gem - uses: trocco-io/push-gem-to-gpr-action@v1 + uses: trocco-io/push-gem-to-gpr-action@v2 with: language: java - gem-path: "./pkg/*.gem" + gem-path: "./build/gems/*.gem" github-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.gitignore b/.gitignore index edc8946..b6eceda 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ build/ /.idea *.iml +*.gemspec diff --git a/Gemfile b/Gemfile deleted file mode 100644 index 9784aee..0000000 --- a/Gemfile +++ /dev/null @@ -1,2 +0,0 @@ -source 'https://rubygems.org/' -gemspec diff --git a/Rakefile b/Rakefile deleted file mode 100644 index 8bfbf16..0000000 --- a/Rakefile +++ /dev/null @@ -1,3 +0,0 @@ -require "bundler/gem_tasks" - -task default: :build diff --git a/build.gradle b/build.gradle index 6d1b9de..a34bb7b 100644 --- a/build.gradle +++ b/build.gradle @@ -1,19 +1,16 @@ plugins { - id "com.jfrog.bintray" version "1.1" - id "com.github.jruby-gradle.base" version "0.1.5" - id "com.palantir.git-version" version "0.13.0" id "java" id "jacoco" + id "com.diffplug.spotless" version "6.11.0" + id "com.palantir.git-version" version "3.4.0" + id "org.embulk.embulk-plugins" version "0.7.0" } -import com.github.jrubygradle.JRubyExec repositories { mavenCentral() - jcenter() -} -configurations { - provided } +description = "JSONL parser plugin for Embulk" + version = { def vd = versionDetails() if (vd.commitDistance == 0 && vd.lastTag ==~ /^[0-9]+\.[0-9]+\.[0-9]+(\.[a-zA-Z0-9]+)?/) { @@ -24,64 +21,54 @@ version = { }() compileJava.options.encoding = 'UTF-8' // source encoding -sourceCompatibility = 1.7 -targetCompatibility = 1.7 - -dependencies { - compile "org.embulk:embulk-core:0.8.8" - provided "org.embulk:embulk-core:0.8.8" +sourceCompatibility = 1.8 +targetCompatibility = 1.8 - testCompile "junit:junit:4.+" - testCompile "org.embulk:embulk-core:0.8.8:tests" - testCompile "org.embulk:embulk-standards:0.8.8" +embulkPlugin { + group = "primenumber-dev" + mainClass = "org.embulk.parser.jsonl.JsonlParserPlugin" + category = "parser" + type = "jsonl" } -task classpath(type: Copy, dependsOn: ["jar"]) { - doFirst { file("classpath").deleteDir() } - from (configurations.runtime - configurations.provided + files(jar.archivePath)) - into "classpath" +gem { + authors = ["Shunsuke Mikami"] + email = ["shun0102@gmail.com"] + summary = "Jsonl parser plugin for Embulk" + homepage = "https://github.com/shun0102/embulk-parser-jsonl" + licenses = ["MIT"] } -clean { delete "classpath" } -task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) { - jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build" - script "${project.name}.gemspec" - doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") } -} +dependencies { + compileOnly "org.embulk:embulk-api:0.10.43" + compileOnly "org.embulk:embulk-spi:0.11" -task gemPush(type: JRubyExec, dependsOn: ["gem"]) { - jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push" - script "pkg/${project.name}-${project.version}.gem" -} + implementation "org.embulk:embulk-util-config:0.3.4" + implementation "org.embulk:embulk-util-json:0.3.0" + implementation "org.embulk:embulk-util-timestamp:0.2.2" + implementation "org.embulk:embulk-util-text:0.1.1" -task "package"(dependsOn: ["gemspec", "classpath"]) << { - println "> Build succeeded." - println "> You can run embulk with '-L ${file(".").absolutePath}' argument." + testImplementation "junit:junit:4.+" + testImplementation "org.embulk:embulk-api:0.10.43" + testImplementation "org.embulk:embulk-spi:0.11" + testImplementation "org.embulk:embulk-junit4:0.11.5" + testImplementation "org.embulk:embulk-util-config:0.3.4" + testImplementation "org.embulk:embulk-deps:0.11.5" } -task gemspec { - ext.gemspecFile = file("${project.name}.gemspec") - inputs.file "build.gradle" - outputs.file gemspecFile - doLast { gemspecFile.write($/ -Gem::Specification.new do |spec| - spec.name = "${project.name}" - spec.version = "${project.version}" - spec.authors = ["Shunsuke Mikami"] - spec.summary = "Jsonl parser plugin for Embulk" - spec.description = "Parses Jsonl files read by other file input plugins." - spec.email = ["shun0102@gmail.com"] - spec.licenses = ["MIT"] - spec.homepage = "https://github.com/shun0102/embulk-parser-jsonl" - - spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"] - spec.test_files = spec.files.grep(%r{^(test|spec)/}) - spec.require_paths = ["lib"] - spec.add_development_dependency 'bundler', ['~> 1.0'] - spec.add_development_dependency 'rake', ['~> 10.0'] -end -/$) +spotless { + java { + importOrder() + removeUnusedImports() + googleJavaFormat() } } -clean { delete "${project.name}.gemspec" } + +dependencyLocking { + lockAllConfigurations() +} + +tasks.withType(JavaCompile) { + options.compilerArgs << "-Xlint:deprecation" +} diff --git a/embulk-parser-jsonl.gemspec b/embulk-parser-jsonl.gemspec deleted file mode 100644 index 1912949..0000000 --- a/embulk-parser-jsonl.gemspec +++ /dev/null @@ -1,18 +0,0 @@ - -Gem::Specification.new do |spec| - spec.name = "embulk-parser-jsonl" - spec.version = "0.2.2" - spec.authors = ["Shunsuke Mikami"] - spec.summary = "Jsonl parser plugin for Embulk" - spec.description = "Parses Jsonl files read by other file input plugins." - spec.email = ["shun0102@gmail.com"] - spec.licenses = ["MIT"] - spec.homepage = "https://github.com/shun0102/embulk-parser-jsonl" - - spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"] - spec.test_files = spec.files.grep(%r{^(test|spec)/}) - spec.require_paths = ["lib"] - - spec.add_development_dependency 'bundler', ['~> 1.0'] - spec.add_development_dependency 'rake', ['~> 10.0'] -end diff --git a/gradle.lockfile b/gradle.lockfile new file mode 100644 index 0000000..8ff5438 --- /dev/null +++ b/gradle.lockfile @@ -0,0 +1,19 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.fasterxml.jackson.core:jackson-annotations:2.6.7=compileClasspath,runtimeClasspath +com.fasterxml.jackson.core:jackson-core:2.6.7=compileClasspath,runtimeClasspath +com.fasterxml.jackson.core:jackson-databind:2.6.7.5=compileClasspath,runtimeClasspath +com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7=compileClasspath,runtimeClasspath +javax.validation:validation-api:1.1.0.Final=compileClasspath,runtimeClasspath +org.embulk:embulk-api:0.10.43=compileClasspath +org.embulk:embulk-spi:0.11=compileClasspath +org.embulk:embulk-util-config:0.3.4=compileClasspath,runtimeClasspath +org.embulk:embulk-util-file:0.1.3=compileClasspath,runtimeClasspath +org.embulk:embulk-util-json:0.3.0=compileClasspath,runtimeClasspath +org.embulk:embulk-util-rubytime:0.3.3=compileClasspath,runtimeClasspath +org.embulk:embulk-util-text:0.1.1=compileClasspath,runtimeClasspath +org.embulk:embulk-util-timestamp:0.2.2=compileClasspath,runtimeClasspath +org.msgpack:msgpack-core:0.8.24=compileClasspath +org.slf4j:slf4j-api:2.0.7=compileClasspath +empty= diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 06b273e..559efb4 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,5 @@ -#Tue Aug 11 00:26:20 PDT 2015 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip diff --git a/lib/embulk/guess/jsonl.rb b/lib/embulk/guess/jsonl.rb deleted file mode 100644 index 9005f5e..0000000 --- a/lib/embulk/guess/jsonl.rb +++ /dev/null @@ -1,58 +0,0 @@ -require 'json' -require "embulk/parser/jsonl.rb" - -module Embulk - module Guess - # $ embulk guess -g "jsonl" partial-config.yml - - class Jsonl < TextGuessPlugin - Plugin.register_guess("jsonl", self) - - def guess_text(config, sample_text) - return {} unless config.dig("parser", "type") == "jsonl" - - rows = [] - - newline_type = config.fetch("parser", {}).fetch("newline", "CRLF") - newline_char = newline_character(newline_type) - sample_text.split(newline_char).each do |line| - next if line.strip.empty? - rows << JSON.parse(line) - end - - min_rows_for_guess = config.fetch("parser", {}).fetch("min_rows_for_guess", 4) - return {} if rows.size < min_rows_for_guess - - if rows.empty? - raise "SchemaGuess Can't guess schema from no records" - end - column_names = rows.map(&:keys).flatten.uniq - samples = rows.to_a.map { |hash| column_names.map { |name| hash[name] } } - - columns = Embulk::Guess::SchemaGuess.from_array_records(column_names, samples).map do |c| - column = { name: c.name, type: c.type } - column[:format] = c.format if c.format - column - end - parser_guessed = {"type" => "jsonl"} - parser_guessed["columns"] = columns - return {"parser" => parser_guessed} - end - - private - - def newline_character(newline_type) - case newline_type - when "CRLF" - "\r\n" - when "LF" - "\n" - when "CR" - "\r" - else - "\r\n" - end - end - end - end -end diff --git a/lib/embulk/parser/jsonl.rb b/lib/embulk/parser/jsonl.rb deleted file mode 100644 index baa14bd..0000000 --- a/lib/embulk/parser/jsonl.rb +++ /dev/null @@ -1,3 +0,0 @@ -Embulk::JavaPlugin.register_parser( - "jsonl", "org.embulk.parser.jsonl.JsonlParserPlugin", - File.expand_path('../../../../classpath', __FILE__)) \ No newline at end of file diff --git a/src/main/java/org/embulk/parser/jsonl/ColumnCaster.java b/src/main/java/org/embulk/parser/jsonl/ColumnCaster.java index 5f8c4be..b526a34 100644 --- a/src/main/java/org/embulk/parser/jsonl/ColumnCaster.java +++ b/src/main/java/org/embulk/parser/jsonl/ColumnCaster.java @@ -1,97 +1,75 @@ package org.embulk.parser.jsonl; +import java.time.Instant; import org.embulk.parser.jsonl.cast.BooleanCast; import org.embulk.parser.jsonl.cast.DoubleCast; import org.embulk.parser.jsonl.cast.JsonCast; import org.embulk.parser.jsonl.cast.LongCast; import org.embulk.parser.jsonl.cast.StringCast; import org.embulk.spi.DataException; -import org.embulk.spi.time.Timestamp; -import org.embulk.spi.time.TimestampParser; +import org.embulk.util.timestamp.TimestampFormatter; import org.msgpack.value.Value; -class ColumnCaster -{ - ColumnCaster() {} +class ColumnCaster { + ColumnCaster() {} - public static boolean asBoolean(Value value) throws DataException - { - if (value.isBooleanValue()) { - return value.asBooleanValue().getBoolean(); - } - else if (value.isIntegerValue()) { - return LongCast.asBoolean(value.asIntegerValue().asLong()); - } - else if (value.isFloatValue()) { - return DoubleCast.asBoolean(value.asFloatValue().toDouble()); - } - else if (value.isStringValue()) { - return StringCast.asBoolean(value.asStringValue().asString()); - } - else { - return JsonCast.asBoolean(value); - } + public static boolean asBoolean(Value value) throws DataException { + if (value.isBooleanValue()) { + return value.asBooleanValue().getBoolean(); + } else if (value.isIntegerValue()) { + return LongCast.asBoolean(value.asIntegerValue().asLong()); + } else if (value.isFloatValue()) { + return DoubleCast.asBoolean(value.asFloatValue().toDouble()); + } else if (value.isStringValue()) { + return StringCast.asBoolean(value.asStringValue().asString()); + } else { + return JsonCast.asBoolean(value); } + } - public static long asLong(Value value) throws DataException - { - if (value.isBooleanValue()) { - return BooleanCast.asLong(value.asBooleanValue().getBoolean()); - } - else if (value.isIntegerValue()) { - return value.asIntegerValue().asLong(); - } - else if (value.isFloatValue()) { - return DoubleCast.asLong(value.asFloatValue().toDouble()); - } - else if (value.isStringValue()) { - return StringCast.asLong(value.asStringValue().asString()); - } - else { - return JsonCast.asLong(value); - } + public static long asLong(Value value) throws DataException { + if (value.isBooleanValue()) { + return BooleanCast.asLong(value.asBooleanValue().getBoolean()); + } else if (value.isIntegerValue()) { + return value.asIntegerValue().asLong(); + } else if (value.isFloatValue()) { + return DoubleCast.asLong(value.asFloatValue().toDouble()); + } else if (value.isStringValue()) { + return StringCast.asLong(value.asStringValue().asString()); + } else { + return JsonCast.asLong(value); } + } - public static double asDouble(Value value) throws DataException - { - if (value.isBooleanValue()) { - return BooleanCast.asDouble(value.asBooleanValue().getBoolean()); - } - else if (value.isIntegerValue()) { - return LongCast.asDouble(value.asIntegerValue().asLong()); - } - else if (value.isFloatValue()) { - return value.asFloatValue().toDouble(); - } - else if (value.isStringValue()) { - return StringCast.asDouble(value.asStringValue().asString()); - } - else { - return JsonCast.asDouble(value); - } + public static double asDouble(Value value) throws DataException { + if (value.isBooleanValue()) { + return BooleanCast.asDouble(value.asBooleanValue().getBoolean()); + } else if (value.isIntegerValue()) { + return LongCast.asDouble(value.asIntegerValue().asLong()); + } else if (value.isFloatValue()) { + return value.asFloatValue().toDouble(); + } else if (value.isStringValue()) { + return StringCast.asDouble(value.asStringValue().asString()); + } else { + return JsonCast.asDouble(value); } + } - public static String asString(Value value) throws DataException - { - return value.toString(); - } + public static String asString(Value value) throws DataException { + return value.toString(); + } - public static Timestamp asTimestamp(Value value, TimestampParser parser) throws DataException - { - if (value.isBooleanValue()) { - return BooleanCast.asTimestamp(value.asBooleanValue().getBoolean()); - } - else if (value.isIntegerValue()) { - return LongCast.asTimestamp(value.asIntegerValue().asLong()); - } - else if (value.isFloatValue()) { - return DoubleCast.asTimestamp(value.asFloatValue().toDouble()); - } - else if (value.isStringValue()) { - return StringCast.asTimestamp(value.asStringValue().asString(), parser); - } - else { - return JsonCast.asTimestamp(value); - } + public static Instant asInstant(Value value, TimestampFormatter formatter) throws DataException { + if (value.isBooleanValue()) { + return BooleanCast.asInstant(value.asBooleanValue().getBoolean()); + } else if (value.isIntegerValue()) { + return LongCast.asInstant(value.asIntegerValue().asLong()); + } else if (value.isFloatValue()) { + return DoubleCast.asInstant(value.asFloatValue().toDouble()); + } else if (value.isStringValue()) { + return StringCast.asInstant(value.asStringValue().asString(), formatter); + } else { + return JsonCast.asInstant(value); } + } } diff --git a/src/main/java/org/embulk/parser/jsonl/ColumnVisitorImpl.java b/src/main/java/org/embulk/parser/jsonl/ColumnVisitorImpl.java index b635923..dcd3fd4 100644 --- a/src/main/java/org/embulk/parser/jsonl/ColumnVisitorImpl.java +++ b/src/main/java/org/embulk/parser/jsonl/ColumnVisitorImpl.java @@ -1,164 +1,173 @@ package org.embulk.parser.jsonl; -import com.google.common.base.Optional; +import java.time.Instant; +import java.util.Optional; import org.embulk.parser.jsonl.JsonlParserPlugin.PluginTask; import org.embulk.parser.jsonl.JsonlParserPlugin.TypecastColumnOption; - import org.embulk.spi.Column; -import org.embulk.spi.ColumnConfig; import org.embulk.spi.ColumnVisitor; import org.embulk.spi.PageBuilder; import org.embulk.spi.Schema; -import org.embulk.spi.SchemaConfig; import org.embulk.spi.time.Timestamp; -import org.embulk.spi.time.TimestampParser; +import org.embulk.util.config.ConfigMapperFactory; +import org.embulk.util.config.units.ColumnConfig; +import org.embulk.util.config.units.SchemaConfig; +import org.embulk.util.timestamp.TimestampFormatter; import org.msgpack.core.MessageTypeException; import org.msgpack.value.Value; public class ColumnVisitorImpl implements ColumnVisitor { - protected final PluginTask task; - protected final Schema schema; - protected final PageBuilder pageBuilder; - protected final TimestampParser[] timestampParsers; - protected final Boolean autoTypecasts[]; + private static final ConfigMapperFactory CONFIG_MAPPER_FACTORY = + ConfigMapperFactory.builder().addDefaultModules().build(); - protected Value value; + protected final PluginTask task; + protected final Schema schema; + protected final PageBuilder pageBuilder; + protected final TimestampFormatter[] timestampFormatters; + protected final Boolean autoTypecasts[]; - public ColumnVisitorImpl(PluginTask task, Schema schema, PageBuilder pageBuilder, TimestampParser[] timestampParsers) - { - this.task = task; - this.schema = schema; - this.pageBuilder = pageBuilder; - this.timestampParsers = timestampParsers; - this.autoTypecasts = new Boolean[schema.size()]; - buildAutoTypecasts(); - } + protected Value value; - private void buildAutoTypecasts() - { - for (Column column : schema.getColumns()) { - this.autoTypecasts[column.getIndex()] = task.getDefaultTypecast(); - } + public ColumnVisitorImpl( + PluginTask task, + Schema schema, + PageBuilder pageBuilder, + TimestampFormatter[] timestampFormatters) { + this.task = task; + this.schema = schema; + this.pageBuilder = pageBuilder; + this.timestampFormatters = timestampFormatters; + this.autoTypecasts = new Boolean[schema.size()]; + buildAutoTypecasts(); + } - Optional schemaConfig = task.getSchemaConfig(); - if (schemaConfig.isPresent()) { - for (ColumnConfig columnConfig : schemaConfig.get().getColumns()) { - TypecastColumnOption columnOption = columnConfig.getOption().loadConfig(TypecastColumnOption.class); - Boolean autoTypecast = columnOption.getTypecast().or(task.getDefaultTypecast()); - Column column = schema.lookupColumn(columnConfig.getName()); - this.autoTypecasts[column.getIndex()] = autoTypecast; - } - } + private void buildAutoTypecasts() { + for (Column column : schema.getColumns()) { + this.autoTypecasts[column.getIndex()] = task.getDefaultTypecast(); } - public void setValue(Value value) - { - this.value = value; + Optional schemaConfig = task.getSchemaConfig(); + if (schemaConfig.isPresent()) { + for (ColumnConfig columnConfig : schemaConfig.get().getColumns()) { + TypecastColumnOption columnOption = + CONFIG_MAPPER_FACTORY + .createConfigMapper() + .map(columnConfig.getOption(), TypecastColumnOption.class); + Boolean autoTypecast = columnOption.getTypecast().orElse(task.getDefaultTypecast()); + Column column = schema.lookupColumn(columnConfig.getName()); + this.autoTypecasts[column.getIndex()] = autoTypecast; + } } + } - @Override - public void booleanColumn(Column column) - { - if (isNil(value)) { - pageBuilder.setNull(column); - } - else { - try { - boolean booleanValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asBoolean(value) : value.asBooleanValue().getBoolean(); - pageBuilder.setBoolean(column, booleanValue); - } - catch (MessageTypeException e) { - throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Boolean", value), e); - } - } - } + public void setValue(Value value) { + this.value = value; + } - @Override - public void longColumn(Column column) - { - if (isNil(value)) { - pageBuilder.setNull(column); - } - else { - try { - long longValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asLong(value) : value.asIntegerValue().toLong(); - pageBuilder.setLong(column, longValue); - } - catch (MessageTypeException e) { - throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Long", value), e); - } - } + @Override + public void booleanColumn(Column column) { + if (isNil(value)) { + pageBuilder.setNull(column); + } else { + try { + boolean booleanValue = + autoTypecasts[column.getIndex()] + ? ColumnCaster.asBoolean(value) + : value.asBooleanValue().getBoolean(); + pageBuilder.setBoolean(column, booleanValue); + } catch (MessageTypeException e) { + throw new JsonRecordValidateException( + String.format("failed to get \"%s\" as Boolean", value), e); + } } + } - @Override - public void doubleColumn(Column column) - { - if (isNil(value)) { - pageBuilder.setNull(column); - } - else { - try { - double doubleValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asDouble(value) : value.asFloatValue().toDouble(); - pageBuilder.setDouble(column, doubleValue); - } - catch (MessageTypeException e) { - throw new JsonRecordValidateException(String.format("failed get \"%s\" as Double", value), e); - } - } + @Override + public void longColumn(Column column) { + if (isNil(value)) { + pageBuilder.setNull(column); + } else { + try { + long longValue = + autoTypecasts[column.getIndex()] + ? ColumnCaster.asLong(value) + : value.asIntegerValue().toLong(); + pageBuilder.setLong(column, longValue); + } catch (MessageTypeException e) { + throw new JsonRecordValidateException( + String.format("failed to get \"%s\" as Long", value), e); + } } + } - @Override - public void stringColumn(Column column) - { - if (isNil(value)) { - pageBuilder.setNull(column); - } - else { - try { - String string = autoTypecasts[column.getIndex()] ? ColumnCaster.asString(value) : value.asStringValue().toString(); - pageBuilder.setString(column, string); - } - catch (MessageTypeException e) { - throw new JsonRecordValidateException(String.format("failed to get \"%s\" as String", value), e); - } - } + @Override + public void doubleColumn(Column column) { + if (isNil(value)) { + pageBuilder.setNull(column); + } else { + try { + double doubleValue = + autoTypecasts[column.getIndex()] + ? ColumnCaster.asDouble(value) + : value.asFloatValue().toDouble(); + pageBuilder.setDouble(column, doubleValue); + } catch (MessageTypeException e) { + throw new JsonRecordValidateException( + String.format("failed get \"%s\" as Double", value), e); + } } + } - @Override - public void timestampColumn(Column column) - { - if (isNil(value)) { - pageBuilder.setNull(column); - } - else { - try { - Timestamp timestamp = ColumnCaster.asTimestamp(value, timestampParsers[column.getIndex()]); - pageBuilder.setTimestamp(column, timestamp); - } - catch (MessageTypeException e) { - throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Timestamp", value), e); - } - } + @Override + public void stringColumn(Column column) { + if (isNil(value)) { + pageBuilder.setNull(column); + } else { + try { + String string = + autoTypecasts[column.getIndex()] + ? ColumnCaster.asString(value) + : value.asStringValue().toString(); + pageBuilder.setString(column, string); + } catch (MessageTypeException e) { + throw new JsonRecordValidateException( + String.format("failed to get \"%s\" as String", value), e); + } } + } - @Override - public void jsonColumn(Column column) - { - if (isNil(value)) { - pageBuilder.setNull(column); - } - else { - try { - pageBuilder.setJson(column, value); - } - catch (MessageTypeException e) { - throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Json", value), e); - } - } + @Override + @SuppressWarnings("deprecation") + public void timestampColumn(Column column) { + if (isNil(value)) { + pageBuilder.setNull(column); + } else { + try { + Instant instant = ColumnCaster.asInstant(value, timestampFormatters[column.getIndex()]); + pageBuilder.setTimestamp(column, Timestamp.ofInstant(instant)); + } catch (MessageTypeException e) { + throw new JsonRecordValidateException( + String.format("failed to get \"%s\" as Timestamp", value), e); + } } + } - protected boolean isNil(Value v) - { - return v == null || v.isNilValue(); + @Override + @SuppressWarnings("deprecation") + public void jsonColumn(Column column) { + if (isNil(value)) { + pageBuilder.setNull(column); + } else { + try { + pageBuilder.setJson(column, value); + } catch (MessageTypeException e) { + throw new JsonRecordValidateException( + String.format("failed to get \"%s\" as Json", value), e); + } } + } + + protected boolean isNil(Value v) { + return v == null || v.isNilValue(); + } } diff --git a/src/main/java/org/embulk/parser/jsonl/JsonRecordValidateException.java b/src/main/java/org/embulk/parser/jsonl/JsonRecordValidateException.java index 1b68efe..f5ec751 100644 --- a/src/main/java/org/embulk/parser/jsonl/JsonRecordValidateException.java +++ b/src/main/java/org/embulk/parser/jsonl/JsonRecordValidateException.java @@ -2,21 +2,16 @@ import org.embulk.spi.DataException; -public class JsonRecordValidateException - extends DataException -{ - public JsonRecordValidateException(String message) - { - super(message); - } +public class JsonRecordValidateException extends DataException { + public JsonRecordValidateException(String message) { + super(message); + } - public JsonRecordValidateException(String message, Throwable cause) - { - super(message, cause); - } + public JsonRecordValidateException(String message, Throwable cause) { + super(message, cause); + } - public JsonRecordValidateException(Throwable cause) - { - super(cause); - } + public JsonRecordValidateException(Throwable cause) { + super(cause); + } } diff --git a/src/main/java/org/embulk/parser/jsonl/JsonlParserPlugin.java b/src/main/java/org/embulk/parser/jsonl/JsonlParserPlugin.java index 8d84b23..8472bd1 100644 --- a/src/main/java/org/embulk/parser/jsonl/JsonlParserPlugin.java +++ b/src/main/java/org/embulk/parser/jsonl/JsonlParserPlugin.java @@ -1,225 +1,270 @@ package org.embulk.parser.jsonl; -import com.google.common.base.Optional; -import com.google.common.base.Supplier; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import org.embulk.config.Config; -import org.embulk.config.ConfigDefault; +import static org.msgpack.value.ValueFactory.newString; + +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; import org.embulk.config.ConfigException; import org.embulk.config.ConfigSource; -import org.embulk.config.Task; import org.embulk.config.TaskSource; import org.embulk.spi.Column; -import org.embulk.spi.ColumnConfig; import org.embulk.spi.DataException; -import org.embulk.spi.Exec; import org.embulk.spi.FileInput; import org.embulk.spi.PageBuilder; import org.embulk.spi.PageOutput; import org.embulk.spi.ParserPlugin; import org.embulk.spi.Schema; -import org.embulk.spi.SchemaConfig; -import org.embulk.spi.json.JsonParseException; -import org.embulk.spi.json.JsonParser; -import org.embulk.spi.time.TimestampParser; import org.embulk.spi.type.Type; -import org.embulk.spi.util.LineDecoder; -import org.embulk.spi.util.Timestamps; +import org.embulk.util.config.Config; +import org.embulk.util.config.ConfigDefault; +import org.embulk.util.config.ConfigMapperFactory; +import org.embulk.util.config.Task; +import org.embulk.util.config.units.ColumnConfig; +import org.embulk.util.config.units.SchemaConfig; +import org.embulk.util.json.JsonParseException; +import org.embulk.util.json.JsonParser; +import org.embulk.util.text.LineDecoder; +import org.embulk.util.text.LineDelimiter; +import org.embulk.util.text.Newline; +import org.embulk.util.timestamp.TimestampFormatter; import org.msgpack.value.Value; import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class JsonlParserPlugin implements ParserPlugin { + private static final ConfigMapperFactory CONFIG_MAPPER_FACTORY = + ConfigMapperFactory.builder().addDefaultModules().build(); + + @Deprecated + public interface JsonlColumnOption extends Task { + @Config("type") + @ConfigDefault("null") + Optional getType(); + } + + public interface TypecastColumnOption extends Task { + @Config("typecast") + @ConfigDefault("null") + Optional getTypecast(); + } + + public interface PluginTask extends Task { + @Config("columns") + @ConfigDefault("null") + Optional getSchemaConfig(); + + @Config("schema") + @ConfigDefault("null") + @Deprecated + Optional getOldSchemaConfig(); -import java.util.Map; + @Config("stop_on_invalid_record") + @ConfigDefault("false") + boolean getStopOnInvalidRecord(); -import static org.msgpack.value.ValueFactory.newString; + @Config("default_typecast") + @ConfigDefault("true") + Boolean getDefaultTypecast(); -public class JsonlParserPlugin - implements ParserPlugin -{ + @Config("column_options") + @ConfigDefault("{}") @Deprecated - public interface JsonlColumnOption - extends Task - { - @Config("type") - @ConfigDefault("null") - Optional getType(); - } + Map getColumnOptions(); - public interface TypecastColumnOption - extends Task - { - @Config("typecast") - @ConfigDefault("null") - public Optional getTypecast(); - } + @Config("charset") + @ConfigDefault("\"utf-8\"") + String getCharset(); - public interface PluginTask - extends Task, LineDecoder.DecoderTask, TimestampParser.Task - { - @Config("columns") - @ConfigDefault("null") - Optional getSchemaConfig(); - - @Config("schema") - @ConfigDefault("null") - @Deprecated - Optional getOldSchemaConfig(); - - @Config("stop_on_invalid_record") - @ConfigDefault("false") - boolean getStopOnInvalidRecord(); - - @Config("default_typecast") - @ConfigDefault("true") - Boolean getDefaultTypecast(); - - @Config("column_options") - @ConfigDefault("{}") - @Deprecated - Map getColumnOptions(); - } + @Config("newline") + @ConfigDefault("\"LF\"") + String getNewline(); - private final Logger log; + @Config("default_timezone") + @ConfigDefault("\"UTC\"") + String getDefaultTimezone(); - private String line = null; - private long lineNumber = 0; - private Map columnNameValues; + @Config("default_timestamp_format") + @ConfigDefault(value = "\"%Y-%m-%d %H:%M:%S.%N %z\"") + String getDefaultTimestampFormat(); + } - public JsonlParserPlugin() - { - this.log = Exec.getLogger(JsonlParserPlugin.class); - } + private static final Logger log = LoggerFactory.getLogger(JsonlParserPlugin.class); - @Override - public void transaction(ConfigSource configSource, Control control) - { - PluginTask task = configSource.loadConfig(PluginTask.class); + private String line = null; + private long lineNumber = 0; + private Map columnNameValues; - if (! task.getColumnOptions().isEmpty()) { - log.warn("embulk-parser-jsonl: \"column_options\" option is deprecated, specify type directly to \"columns\" option with typecast: true (default: true)."); - } + public JsonlParserPlugin() {} - SchemaConfig schemaConfig = getSchemaConfig(task); - ImmutableList.Builder columns = ImmutableList.builder(); - for (int i = 0; i < schemaConfig.getColumnCount(); i++) { - ColumnConfig columnConfig = schemaConfig.getColumn(i); - Type type = getType(task, columnConfig); - columns.add(new Column(i, columnConfig.getName(), type)); - } - control.run(task.dump(), new Schema(columns.build())); - } + @SuppressWarnings("deprecation") + @Override + public void transaction(ConfigSource configSource, Control control) { + final PluginTask task = + CONFIG_MAPPER_FACTORY.createConfigMapper().map(configSource, PluginTask.class); - private static Type getType(PluginTask task, ColumnConfig columnConfig) - { - JsonlColumnOption columnOption = columnOptionOf(task.getColumnOptions(), columnConfig.getName()); - return columnOption.getType().or(columnConfig.getType()); + Map columnOptions = task.getColumnOptions(); + if (columnOptions != null && !columnOptions.isEmpty()) { + log.warn( + "embulk-parser-jsonl: \"column_options\" option is deprecated, specify type directly to \"columns\" option with typecast: true (default: true)."); } - // this method is to keep the backward compatibility of 'schema' option. - private SchemaConfig getSchemaConfig(PluginTask task) - { - if (task.getOldSchemaConfig().isPresent()) { - log.warn("Please use 'columns' option instead of 'schema' because the 'schema' option is deprecated. The next version will stop 'schema' option support."); - } + SchemaConfig schemaConfig = getSchemaConfig(task); + List columns = new ArrayList<>(); + for (int i = 0; i < schemaConfig.getColumnCount(); i++) { + ColumnConfig columnConfig = schemaConfig.getColumn(i); + Type type = getType(task, columnConfig); + columns.add(new Column(i, columnConfig.getName(), type)); + } + control.run(task.dump(), new Schema(columns)); + } + + private static Type getType(PluginTask task, ColumnConfig columnConfig) { + JsonlColumnOption columnOption = + columnOptionOf(task.getColumnOptions(), columnConfig.getName()); + return columnOption.getType().orElse(columnConfig.getType()); + } + + // this method is to keep the backward compatibility of 'schema' option. + private SchemaConfig getSchemaConfig(PluginTask task) { + if (task.getOldSchemaConfig().isPresent()) { + log.warn( + "Please use 'columns' option instead of 'schema' because the 'schema' option is deprecated. The next version will stop 'schema' option support."); + } - if (task.getSchemaConfig().isPresent()) { - return task.getSchemaConfig().get(); - } - else if (task.getOldSchemaConfig().isPresent()) { - return task.getOldSchemaConfig().get(); - } - else { - throw new ConfigException("Attribute 'columns' is required but not set"); - } + if (task.getSchemaConfig().isPresent()) { + return task.getSchemaConfig().get(); + } else if (task.getOldSchemaConfig().isPresent()) { + return task.getOldSchemaConfig().get(); + } else { + throw new ConfigException("Attribute 'columns' is required but not set"); } + } + + @SuppressWarnings("deprecation") + @Override + public void run(TaskSource taskSource, Schema schema, FileInput input, PageOutput output) { + final PluginTask task = + CONFIG_MAPPER_FACTORY.createTaskMapper().map(taskSource, PluginTask.class); + + setColumnNameValues(schema); + + final SchemaConfig schemaConfig = getSchemaConfig(task); + final TimestampFormatter[] timestampFormatters = newTimestampFormatters(task, schemaConfig); + final Charset charset = Charset.forName(task.getCharset()); + final Newline newline = Newline.valueOf(task.getNewline()); + final LineDelimiter lineDelimiter = newlineToLineDelimiter(newline); + final LineDecoder decoder = LineDecoder.of(input, charset, lineDelimiter); + final JsonParser jsonParser = newJsonParser(); + final boolean stopOnInvalidRecord = task.getStopOnInvalidRecord(); + + try (final PageBuilder pageBuilder = + new PageBuilder(org.embulk.spi.Exec.getBufferAllocator(), schema, output)) { + ColumnVisitorImpl visitor = + new ColumnVisitorImpl(task, schema, pageBuilder, timestampFormatters); + + lineNumber = 0; + while (decoder.nextFile()) { + while (true) { + line = decoder.poll(); + if (line == null) { + break; + } + lineNumber++; + + try { + Value value = jsonParser.parse(line); + + if (!value.isMapValue()) { + throw new JsonRecordValidateException("Json string is not representing map value."); + } - @Override - public void run(TaskSource taskSource, Schema schema, FileInput input, PageOutput output) - { - PluginTask task = taskSource.loadTask(PluginTask.class); - - setColumnNameValues(schema); - - final SchemaConfig schemaConfig = getSchemaConfig(task); - final TimestampParser[] timestampParsers = Timestamps.newTimestampColumnParsers(task, schemaConfig); - final LineDecoder decoder = newLineDecoder(input, task); - final JsonParser jsonParser = newJsonParser(); - final boolean stopOnInvalidRecord = task.getStopOnInvalidRecord(); - - try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) { - ColumnVisitorImpl visitor = new ColumnVisitorImpl(task, schema, pageBuilder, timestampParsers); - - while (decoder.nextFile()) { // TODO this implementation should be improved with new JsonParser API on Embulk v0.8.3 - lineNumber = 0; - - while ((line = decoder.poll()) != null) { - lineNumber++; - - try { - Value value = jsonParser.parse(line); - - if (!value.isMapValue()) { - throw new JsonRecordValidateException("Json string is not representing map value."); - } - - final Map record = value.asMapValue().map(); - for (Column column : schema.getColumns()) { - Value v = record.get(getColumnNameValue(column)); - visitor.setValue(v); - column.visit(visitor); - } - - pageBuilder.addRecord(); - } - catch (JsonRecordValidateException | JsonParseException e) { - if (stopOnInvalidRecord) { - throw new DataException(String.format("Invalid record at line %d: %s", lineNumber, line), e); - } - log.warn(String.format("Skipped line %d (%s): %s", lineNumber, e.getMessage(), line)); - } - } + final Map record = value.asMapValue().map(); + for (Column column : schema.getColumns()) { + Value v = record.get(getColumnNameValue(column)); + visitor.setValue(v); + column.visit(visitor); } - pageBuilder.finish(); + pageBuilder.addRecord(); + } catch (JsonRecordValidateException | JsonParseException e) { + if (stopOnInvalidRecord) { + throw new DataException( + String.format("Invalid record at line %d: %s", lineNumber, line), e); + } + log.warn(String.format("Skipped line %d (%s): %s", lineNumber, e.getMessage(), line)); + } } - } + } - private void setColumnNameValues(Schema schema) - { - ImmutableMap.Builder builder = ImmutableMap.builder(); - for (Column column : schema.getColumns()) { - String name = column.getName(); - builder.put(name, newString(name)); - } - columnNameValues = builder.build(); + pageBuilder.finish(); } + } - private Value getColumnNameValue(Column column) - { - return columnNameValues.get(column.getName()); + private void setColumnNameValues(Schema schema) { + Map builder = new HashMap<>(); + for (Column column : schema.getColumns()) { + String name = column.getName(); + builder.put(name, newString(name)); } - - public LineDecoder newLineDecoder(FileInput input, PluginTask task) - { - return new LineDecoder(input, task); + columnNameValues = builder; + } + + private Value getColumnNameValue(Column column) { + return columnNameValues.get(column.getName()); + } + + @SuppressWarnings("deprecation") + public JsonParser newJsonParser() { + return new JsonParser(); + } + + private static LineDelimiter newlineToLineDelimiter(Newline newline) { + switch (newline) { + case CR: + return LineDelimiter.CR; + case LF: + return LineDelimiter.LF; + case CRLF: + return LineDelimiter.CRLF; + default: + return LineDelimiter.CRLF; } - - public JsonParser newJsonParser() - { - return new JsonParser(); + } + + private TimestampFormatter[] newTimestampFormatters(PluginTask task, SchemaConfig schemaConfig) { + TimestampFormatter[] formatters = new TimestampFormatter[schemaConfig.getColumnCount()]; + int i = 0; + for (ColumnConfig columnConfig : schemaConfig.getColumns()) { + if (columnConfig.getType() instanceof org.embulk.spi.type.TimestampType) { + String pattern = + columnConfig.getOption().get(String.class, "format", task.getDefaultTimestampFormat()); + formatters[i] = + TimestampFormatter.builder(pattern, true) + .setDefaultZoneFromString(task.getDefaultTimezone()) + .build(); + } + i++; } - - private static JsonlColumnOption columnOptionOf(Map columnOptions, String columnName) - { - return Optional.fromNullable(columnOptions.get(columnName)).or( - // default column option - new Supplier() - { - public JsonlColumnOption get() - { - return Exec.newConfigSource().loadConfig(JsonlColumnOption.class); - } - }); + return formatters; + } + + private static JsonlColumnOption columnOptionOf( + Map columnOptions, String columnName) { + if (columnOptions == null) { + return CONFIG_MAPPER_FACTORY + .createConfigMapper() + .map(CONFIG_MAPPER_FACTORY.newConfigSource(), JsonlColumnOption.class); } - + JsonlColumnOption option = columnOptions.get(columnName); + if (option != null) { + return option; + } + return CONFIG_MAPPER_FACTORY + .createConfigMapper() + .map(CONFIG_MAPPER_FACTORY.newConfigSource(), JsonlColumnOption.class); + } } diff --git a/src/main/java/org/embulk/parser/jsonl/cast/BooleanCast.java b/src/main/java/org/embulk/parser/jsonl/cast/BooleanCast.java index 8f946e9..b5c516b 100644 --- a/src/main/java/org/embulk/parser/jsonl/cast/BooleanCast.java +++ b/src/main/java/org/embulk/parser/jsonl/cast/BooleanCast.java @@ -1,39 +1,32 @@ package org.embulk.parser.jsonl.cast; +import java.time.Instant; import org.embulk.spi.DataException; -import org.embulk.spi.time.Timestamp; - -public class BooleanCast -{ - private BooleanCast() {} - - private static String buildErrorMessage(String as, boolean value) - { - return String.format("cannot cast boolean to %s: \"%s\"", as, value); - } - - public static boolean asBoolean(boolean value) throws DataException - { - return value; - } - - public static long asLong(boolean value) throws DataException - { - return value ? 1 : 0; - } - - public static double asDouble(boolean value) throws DataException - { - throw new DataException(buildErrorMessage("double", value)); - } - - public static String asString(boolean value) throws DataException - { - return value ? "true" : "false"; - } - - public static Timestamp asTimestamp(boolean value) throws DataException - { - throw new DataException(buildErrorMessage("timestamp", value)); - } + +public class BooleanCast { + private BooleanCast() {} + + private static String buildErrorMessage(String as, boolean value) { + return String.format("cannot cast boolean to %s: \"%s\"", as, value); + } + + public static boolean asBoolean(boolean value) throws DataException { + return value; + } + + public static long asLong(boolean value) throws DataException { + return value ? 1 : 0; + } + + public static double asDouble(boolean value) throws DataException { + throw new DataException(buildErrorMessage("double", value)); + } + + public static String asString(boolean value) throws DataException { + return value ? "true" : "false"; + } + + public static Instant asInstant(boolean value) throws DataException { + throw new DataException(buildErrorMessage("timestamp", value)); + } } diff --git a/src/main/java/org/embulk/parser/jsonl/cast/DoubleCast.java b/src/main/java/org/embulk/parser/jsonl/cast/DoubleCast.java index 8740623..1e2926f 100644 --- a/src/main/java/org/embulk/parser/jsonl/cast/DoubleCast.java +++ b/src/main/java/org/embulk/parser/jsonl/cast/DoubleCast.java @@ -1,41 +1,34 @@ package org.embulk.parser.jsonl.cast; +import java.time.Instant; import org.embulk.spi.DataException; -import org.embulk.spi.time.Timestamp; - -public class DoubleCast -{ - private DoubleCast() {} - - private static String buildErrorMessage(String as, double value) - { - return String.format("cannot cast double to %s: \"%s\"", as, value); - } - - public static boolean asBoolean(double value) throws DataException - { - throw new DataException(buildErrorMessage("boolean", value)); - } - - public static long asLong(double value) throws DataException - { - return (long) value; - } - - public static double asDouble(double value) throws DataException - { - return value; - } - - public static String asString(double value) throws DataException - { - return String.valueOf(value); - } - - public static Timestamp asTimestamp(double value) throws DataException - { - long epochSecond = (long) value; - long nanoAdjustMent = (long) ((value - epochSecond) * 1000000000); - return Timestamp.ofEpochSecond(epochSecond, nanoAdjustMent); - } + +public class DoubleCast { + private DoubleCast() {} + + private static String buildErrorMessage(String as, double value) { + return String.format("cannot cast double to %s: \"%s\"", as, value); + } + + public static boolean asBoolean(double value) throws DataException { + throw new DataException(buildErrorMessage("boolean", value)); + } + + public static long asLong(double value) throws DataException { + return (long) value; + } + + public static double asDouble(double value) throws DataException { + return value; + } + + public static String asString(double value) throws DataException { + return String.valueOf(value); + } + + public static Instant asInstant(double value) throws DataException { + long epochSecond = (long) value; + long nanoAdjustment = (long) ((value - epochSecond) * 1000000000); + return Instant.ofEpochSecond(epochSecond, nanoAdjustment); + } } diff --git a/src/main/java/org/embulk/parser/jsonl/cast/JsonCast.java b/src/main/java/org/embulk/parser/jsonl/cast/JsonCast.java index 2c9e604..ffa0a15 100644 --- a/src/main/java/org/embulk/parser/jsonl/cast/JsonCast.java +++ b/src/main/java/org/embulk/parser/jsonl/cast/JsonCast.java @@ -1,40 +1,33 @@ package org.embulk.parser.jsonl.cast; +import java.time.Instant; import org.embulk.spi.DataException; -import org.embulk.spi.time.Timestamp; import org.msgpack.value.Value; -public class JsonCast -{ - private JsonCast() {} - - private static String buildErrorMessage(String as, Value value) - { - return String.format("cannot cast Json to %s: \"%s\"", as, value); - } - - public static boolean asBoolean(Value value) throws DataException - { - throw new DataException(buildErrorMessage("boolean", value)); - } - - public static long asLong(Value value) throws DataException - { - throw new DataException(buildErrorMessage("long", value)); - } - - public static double asDouble(Value value) throws DataException - { - throw new DataException(buildErrorMessage("double", value)); - } - - public static String asString(Value value) throws DataException - { - return value.toString(); - } - - public static Timestamp asTimestamp(Value value) throws DataException - { - throw new DataException(buildErrorMessage("timestamp", value)); - } +public class JsonCast { + private JsonCast() {} + + private static String buildErrorMessage(String as, Value value) { + return String.format("cannot cast Json to %s: \"%s\"", as, value); + } + + public static boolean asBoolean(Value value) throws DataException { + throw new DataException(buildErrorMessage("boolean", value)); + } + + public static long asLong(Value value) throws DataException { + throw new DataException(buildErrorMessage("long", value)); + } + + public static double asDouble(Value value) throws DataException { + throw new DataException(buildErrorMessage("double", value)); + } + + public static String asString(Value value) throws DataException { + return value.toString(); + } + + public static Instant asInstant(Value value) throws DataException { + throw new DataException(buildErrorMessage("timestamp", value)); + } } diff --git a/src/main/java/org/embulk/parser/jsonl/cast/LongCast.java b/src/main/java/org/embulk/parser/jsonl/cast/LongCast.java index f84ca7d..7639e14 100644 --- a/src/main/java/org/embulk/parser/jsonl/cast/LongCast.java +++ b/src/main/java/org/embulk/parser/jsonl/cast/LongCast.java @@ -1,47 +1,38 @@ package org.embulk.parser.jsonl.cast; +import java.time.Instant; import org.embulk.spi.DataException; -import org.embulk.spi.time.Timestamp; -public class LongCast -{ - private LongCast() {} +public class LongCast { + private LongCast() {} - private static String buildErrorMessage(String as, long value) - { - return String.format("cannot cast long to %s: \"%s\"", as, value); - } + private static String buildErrorMessage(String as, long value) { + return String.format("cannot cast long to %s: \"%s\"", as, value); + } - public static boolean asBoolean(long value) throws DataException - { - if (value == 1) { - return true; - } - else if (value == 0) { - return false; - } - else { - throw new DataException(buildErrorMessage("boolean", value)); - } + public static boolean asBoolean(long value) throws DataException { + if (value == 1) { + return true; + } else if (value == 0) { + return false; + } else { + throw new DataException(buildErrorMessage("boolean", value)); } + } - public static long asLong(long value) throws DataException - { - return value; - } + public static long asLong(long value) throws DataException { + return value; + } - public static double asDouble(long value) throws DataException - { - return (double) value; - } + public static double asDouble(long value) throws DataException { + return (double) value; + } - public static String asString(long value) throws DataException - { - return String.valueOf(value); - } + public static String asString(long value) throws DataException { + return String.valueOf(value); + } - public static Timestamp asTimestamp(long value) throws DataException - { - return Timestamp.ofEpochSecond(value); - } + public static Instant asInstant(long value) throws DataException { + return Instant.ofEpochSecond(value); + } } diff --git a/src/main/java/org/embulk/parser/jsonl/cast/StringCast.java b/src/main/java/org/embulk/parser/jsonl/cast/StringCast.java index 08eebf4..576ed62 100644 --- a/src/main/java/org/embulk/parser/jsonl/cast/StringCast.java +++ b/src/main/java/org/embulk/parser/jsonl/cast/StringCast.java @@ -1,82 +1,68 @@ package org.embulk.parser.jsonl.cast; -import com.google.common.collect.ImmutableSet; +import java.time.Instant; +import java.time.format.DateTimeParseException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; import org.embulk.spi.DataException; -import org.embulk.spi.time.Timestamp; -import org.embulk.spi.time.TimestampParseException; -import org.embulk.spi.time.TimestampParser; +import org.embulk.util.timestamp.TimestampFormatter; -public class StringCast -{ - // copy from csv plugin - public static final ImmutableSet TRUE_STRINGS = - ImmutableSet.of( - "true", "True", "TRUE", - "yes", "Yes", "YES", - "t", "T", "y", "Y", - "on", "On", "ON", - "1"); +public class StringCast { + // copy from csv plugin + public static final Set TRUE_STRINGS = + new HashSet<>( + Arrays.asList( + "true", "True", "TRUE", "yes", "Yes", "YES", "t", "T", "y", "Y", "on", "On", "ON", + "1")); - public static final ImmutableSet FALSE_STRINGS = - ImmutableSet.of( - "false", "False", "FALSE", - "no", "No", "NO", - "f", "F", "n", "N", - "off", "Off", "OFF", - "0"); + public static final Set FALSE_STRINGS = + new HashSet<>( + Arrays.asList( + "false", "False", "FALSE", "no", "No", "NO", "f", "F", "n", "N", "off", "Off", "OFF", + "0")); - private StringCast() {} + private StringCast() {} - private static String buildErrorMessage(String as, String value) - { - return String.format("cannot cast String to %s: \"%s\"", as, value); - } + private static String buildErrorMessage(String as, String value) { + return String.format("cannot cast String to %s: \"%s\"", as, value); + } - public static boolean asBoolean(String value) throws DataException - { - if (TRUE_STRINGS.contains(value)) { - return true; - } - else if (FALSE_STRINGS.contains(value)) { - return false; - } - else { - throw new DataException(buildErrorMessage("boolean", value)); - } + public static boolean asBoolean(String value) throws DataException { + if (TRUE_STRINGS.contains(value)) { + return true; + } else if (FALSE_STRINGS.contains(value)) { + return false; + } else { + throw new DataException(buildErrorMessage("boolean", value)); } + } - public static long asLong(String value) throws DataException - { - try { - return Long.parseLong(value); - } - catch (NumberFormatException ex) { - throw new DataException(buildErrorMessage("long", value), ex); - } + public static long asLong(String value) throws DataException { + try { + return Long.parseLong(value); + } catch (NumberFormatException ex) { + throw new DataException(buildErrorMessage("long", value), ex); } + } - public static double asDouble(String value) throws DataException - { - try { - return Double.parseDouble(value); - } - catch (NumberFormatException ex) { - throw new DataException(buildErrorMessage("double", value), ex); - } + public static double asDouble(String value) throws DataException { + try { + return Double.parseDouble(value); + } catch (NumberFormatException ex) { + throw new DataException(buildErrorMessage("double", value), ex); } + } - public static String asString(String value) throws DataException - { - return value; - } + public static String asString(String value) throws DataException { + return value; + } - public static Timestamp asTimestamp(String value, TimestampParser parser) throws DataException - { - try { - return parser.parse(value); - } - catch (TimestampParseException ex) { - throw new DataException(buildErrorMessage("timestamp", value), ex); - } + public static Instant asInstant(String value, TimestampFormatter formatter) throws DataException { + try { + return formatter.parse(value); + } catch (DateTimeParseException ex) { + throw new DataException(buildErrorMessage("timestamp", value), ex); } + } } diff --git a/src/test/java/org/embulk/parser/jsonl/TestColumnCaster.java b/src/test/java/org/embulk/parser/jsonl/TestColumnCaster.java index f2345ea..e034048 100644 --- a/src/test/java/org/embulk/parser/jsonl/TestColumnCaster.java +++ b/src/test/java/org/embulk/parser/jsonl/TestColumnCaster.java @@ -1,256 +1,212 @@ package org.embulk.parser.jsonl; -import org.embulk.EmbulkTestRuntime; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.time.Instant; import org.embulk.spi.DataException; -import org.embulk.spi.time.Timestamp; -import org.embulk.spi.time.TimestampParser; -import org.joda.time.DateTimeZone; -import org.jruby.embed.ScriptingContainer; +import org.embulk.util.timestamp.TimestampFormatter; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import org.msgpack.value.MapValue; -import org.msgpack.value.Value; import org.msgpack.value.ValueFactory; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -public class TestColumnCaster -{ - @Rule - public EmbulkTestRuntime runtime = new EmbulkTestRuntime(); - public MapValue mapValue; - public DataException thrown; - public ScriptingContainer jruby; - public TimestampParser parser; - - @Before - public void createResource() - { - jruby = new ScriptingContainer(); - thrown = new DataException("any"); - Value[] kvs = new Value[2]; - kvs[0] = ValueFactory.newString("k"); - kvs[1] = ValueFactory.newString("v"); - mapValue = ValueFactory.newMap(kvs); - parser = new TimestampParser(jruby, "%Y-%m-%d %H:%M:%S.%N", DateTimeZone.UTC); - } - - @Test - public void asBooleanFromBoolean() - { - assertEquals(true, ColumnCaster.asBoolean(ValueFactory.newBoolean(true))); - } - - @Test - public void asBooleanFromInteger() - { - assertEquals(true, ColumnCaster.asBoolean(ValueFactory.newInteger(1))); - try { - ColumnCaster.asBoolean(ValueFactory.newInteger(2)); - fail(); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } - } - - @Test - public void asBooleanFromFloat() - { - try { - ColumnCaster.asBoolean(ValueFactory.newFloat(1.1)); - fail(); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } - } - - @Test - public void asBooleanFromString() - { - assertEquals(true, ColumnCaster.asBoolean(ValueFactory.newString("true"))); - try { - ColumnCaster.asBoolean(ValueFactory.newString("foo")); - fail(); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } - } - - @Test - public void asBooleanFromJson() - { - try { - ColumnCaster.asBoolean(mapValue); - fail(); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } - } - - @Test - public void asLongFromBoolean() - { - assertEquals(1, ColumnCaster.asLong(ValueFactory.newBoolean(true))); - } - - @Test - public void asLongFromInteger() - { - assertEquals(1, ColumnCaster.asLong(ValueFactory.newInteger(1))); - } - - @Test - public void asLongFromFloat() - { - assertEquals(1, ColumnCaster.asLong(ValueFactory.newFloat(1.5))); - } - - @Test - public void asLongFromString() - { - assertEquals(1, ColumnCaster.asLong(ValueFactory.newString("1"))); - try { - ColumnCaster.asLong(ValueFactory.newString("foo")); - fail(); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } - } - - @Test - public void asLongFromJson() - { - try { - ColumnCaster.asLong(mapValue); - fail(); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } - } - - @Test - public void asDoubleFromBoolean() - { - assertEquals(1, ColumnCaster.asLong(ValueFactory.newBoolean(true))); - } - - @Test - public void asDoubleFromInteger() - { - assertEquals(1, ColumnCaster.asLong(ValueFactory.newInteger(1))); - } - - @Test - public void asDoubleFromFloat() - { - assertEquals(1, ColumnCaster.asLong(ValueFactory.newFloat(1.5))); - } - - @Test - public void asDoubleFromString() - { - assertEquals(1, ColumnCaster.asLong(ValueFactory.newString("1"))); - try { - ColumnCaster.asLong(ValueFactory.newString("foo")); - fail(); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } - } - - @Test - public void asDoubleFromJson() - { - try { - ColumnCaster.asLong(mapValue); - fail(); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } - } - - @Test - public void asStringFromBoolean() - { - assertEquals("true", ColumnCaster.asString(ValueFactory.newBoolean(true))); - } - - @Test - public void asStringFromInteger() - { - assertEquals("1", ColumnCaster.asString(ValueFactory.newInteger(1))); - } - - @Test - public void asStringFromFloat() - { - assertEquals("1.5", ColumnCaster.asString(ValueFactory.newFloat(1.5))); - } - - @Test - public void asStringFromString() - { - assertEquals("1", ColumnCaster.asString(ValueFactory.newString("1"))); - } - - @Test - public void asStringFromJson() - { - assertEquals("{\"k\":\"v\"}", ColumnCaster.asString(mapValue)); - } - - @Test - public void asTimestampFromBoolean() - { - try { - ColumnCaster.asTimestamp(ValueFactory.newBoolean(true), parser); - fail(); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } - } - - @Test - public void asTimestampFromInteger() - { - assertEquals(1, ColumnCaster.asTimestamp(ValueFactory.newInteger(1), parser).getEpochSecond()); - } - - @Test - public void asTimestampFromFloat() - { - Timestamp expected = Timestamp.ofEpochSecond(1463084053, 500000000); - assertEquals(expected, ColumnCaster.asTimestamp(ValueFactory.newFloat(1463084053.5), parser)); - } - - @Test - public void asTimestampFromString() - { - Timestamp expected = Timestamp.ofEpochSecond(1463084053, 500000000); - assertEquals(expected, ColumnCaster.asTimestamp(ValueFactory.newString("2016-05-12 20:14:13.5"), parser)); - } - - @Test - public void asTimestampFromJson() - { - try { - ColumnCaster.asTimestamp(mapValue, parser); - fail(); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } - } +public class TestColumnCaster { + public MapValue mapValue; + public DataException thrown; + public TimestampFormatter formatter; + + @Before + public void createResource() { + thrown = new DataException("any"); + org.msgpack.value.Value[] kvs = new org.msgpack.value.Value[2]; + kvs[0] = ValueFactory.newString("k"); + kvs[1] = ValueFactory.newString("v"); + mapValue = ValueFactory.newMap(kvs); + formatter = TimestampFormatter.builder("%Y-%m-%d %H:%M:%S.%N", true).build(); + } + + @Test + public void asBooleanFromBoolean() { + assertEquals(true, ColumnCaster.asBoolean(ValueFactory.newBoolean(true))); + } + + @Test + public void asBooleanFromInteger() { + assertEquals(true, ColumnCaster.asBoolean(ValueFactory.newInteger(1))); + try { + ColumnCaster.asBoolean(ValueFactory.newInteger(2)); + fail(); + } catch (Throwable t) { + assertTrue(t instanceof DataException); + } + } + + @Test + public void asBooleanFromFloat() { + try { + ColumnCaster.asBoolean(ValueFactory.newFloat(1.1)); + fail(); + } catch (Throwable t) { + assertTrue(t instanceof DataException); + } + } + + @Test + public void asBooleanFromString() { + assertEquals(true, ColumnCaster.asBoolean(ValueFactory.newString("true"))); + try { + ColumnCaster.asBoolean(ValueFactory.newString("foo")); + fail(); + } catch (Throwable t) { + assertTrue(t instanceof DataException); + } + } + + @Test + public void asBooleanFromJson() { + try { + ColumnCaster.asBoolean(mapValue); + fail(); + } catch (Throwable t) { + assertTrue(t instanceof DataException); + } + } + + @Test + public void asLongFromBoolean() { + assertEquals(1, ColumnCaster.asLong(ValueFactory.newBoolean(true))); + } + + @Test + public void asLongFromInteger() { + assertEquals(1, ColumnCaster.asLong(ValueFactory.newInteger(1))); + } + + @Test + public void asLongFromFloat() { + assertEquals(1, ColumnCaster.asLong(ValueFactory.newFloat(1.5))); + } + + @Test + public void asLongFromString() { + assertEquals(1, ColumnCaster.asLong(ValueFactory.newString("1"))); + try { + ColumnCaster.asLong(ValueFactory.newString("foo")); + fail(); + } catch (Throwable t) { + assertTrue(t instanceof DataException); + } + } + + @Test + public void asLongFromJson() { + try { + ColumnCaster.asLong(mapValue); + fail(); + } catch (Throwable t) { + assertTrue(t instanceof DataException); + } + } + + @Test + public void asDoubleFromBoolean() { + assertEquals(1, ColumnCaster.asLong(ValueFactory.newBoolean(true))); + } + + @Test + public void asDoubleFromInteger() { + assertEquals(1, ColumnCaster.asLong(ValueFactory.newInteger(1))); + } + + @Test + public void asDoubleFromFloat() { + assertEquals(1, ColumnCaster.asLong(ValueFactory.newFloat(1.5))); + } + + @Test + public void asDoubleFromString() { + assertEquals(1, ColumnCaster.asLong(ValueFactory.newString("1"))); + try { + ColumnCaster.asLong(ValueFactory.newString("foo")); + fail(); + } catch (Throwable t) { + assertTrue(t instanceof DataException); + } + } + + @Test + public void asDoubleFromJson() { + try { + ColumnCaster.asLong(mapValue); + fail(); + } catch (Throwable t) { + assertTrue(t instanceof DataException); + } + } + + @Test + public void asStringFromBoolean() { + assertEquals("true", ColumnCaster.asString(ValueFactory.newBoolean(true))); + } + + @Test + public void asStringFromInteger() { + assertEquals("1", ColumnCaster.asString(ValueFactory.newInteger(1))); + } + + @Test + public void asStringFromFloat() { + assertEquals("1.5", ColumnCaster.asString(ValueFactory.newFloat(1.5))); + } + + @Test + public void asStringFromString() { + assertEquals("1", ColumnCaster.asString(ValueFactory.newString("1"))); + } + + @Test + public void asStringFromJson() { + assertEquals("{\"k\":\"v\"}", ColumnCaster.asString(mapValue)); + } + + @Test + public void asInstantFromBoolean() { + try { + ColumnCaster.asInstant(ValueFactory.newBoolean(true), formatter); + fail(); + } catch (Throwable t) { + assertTrue(t instanceof DataException); + } + } + + @Test + public void asInstantFromInteger() { + assertEquals(1, ColumnCaster.asInstant(ValueFactory.newInteger(1), formatter).getEpochSecond()); + } + + @Test + public void asInstantFromFloat() { + Instant expected = Instant.ofEpochSecond(1463084053, 500000000); + assertEquals(expected, ColumnCaster.asInstant(ValueFactory.newFloat(1463084053.5), formatter)); + } + + @Test + public void asInstantFromString() { + Instant expected = Instant.ofEpochSecond(1463084053, 500000000); + assertEquals( + expected, + ColumnCaster.asInstant(ValueFactory.newString("2016-05-12 20:14:13.5"), formatter)); + } + + @Test + public void asInstantFromJson() { + try { + ColumnCaster.asInstant(mapValue, formatter); + fail(); + } catch (Throwable t) { + assertTrue(t instanceof DataException); + } + } } diff --git a/src/test/java/org/embulk/parser/jsonl/TestJsonlParserPlugin.java b/src/test/java/org/embulk/parser/jsonl/TestJsonlParserPlugin.java index 23b1d9e..8e8188f 100644 --- a/src/test/java/org/embulk/parser/jsonl/TestJsonlParserPlugin.java +++ b/src/test/java/org/embulk/parser/jsonl/TestJsonlParserPlugin.java @@ -1,33 +1,5 @@ package org.embulk.parser.jsonl; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import org.embulk.EmbulkTestRuntime; -import org.embulk.config.ConfigLoader; -import org.embulk.config.ConfigSource; -import org.embulk.config.TaskSource; -import org.embulk.spi.ColumnConfig; -import org.embulk.spi.DataException; -import org.embulk.spi.Exec; -import org.embulk.spi.FileInput; -import org.embulk.spi.ParserPlugin; -import org.embulk.spi.Schema; -import org.embulk.spi.SchemaConfig; -import org.embulk.spi.TestPageBuilderReader.MockPageOutput; -import org.embulk.spi.time.Timestamp; -import org.embulk.spi.type.Type; -import org.embulk.spi.util.InputStreamFileInput; -import org.embulk.spi.util.Pages; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; - import static org.embulk.spi.type.Types.BOOLEAN; import static org.embulk.spi.type.Types.DOUBLE; import static org.embulk.spi.type.Types.JSON; @@ -42,237 +14,346 @@ import static org.msgpack.value.ValueFactory.newMap; import static org.msgpack.value.ValueFactory.newString; -public class TestJsonlParserPlugin -{ - @Rule - public EmbulkTestRuntime runtime = new EmbulkTestRuntime(); - - private ConfigSource config; - private JsonlParserPlugin plugin; - private MockPageOutput output; - - @Before - public void createResource() - { - config = config().set("type", "jsonl"); - plugin = new JsonlParserPlugin(); - recreatePageOutput(); - } - - private void recreatePageOutput() - { - output = new MockPageOutput(); - } - - @Test - public void skipRecords() - throws Exception - { - SchemaConfig schema = schema( - column("_c0", BOOLEAN), column("_c1", LONG), column("_c2", DOUBLE), - column("_c3", STRING), column("_c4", TIMESTAMP), column("_c5", JSON)); - ConfigSource config = this.config.deepCopy().set("columns", schema); - - transaction(config, fileInput( - "[]", - "\"embulk\"", - "10", - "true", - "false", - "null", - " " - )); - - List records = Pages.toObjects(schema.toSchema(), output.pages); - assertEquals(0, records.size()); - } +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.embulk.config.ConfigSource; +import org.embulk.config.TaskSource; +import org.embulk.spi.Column; +import org.embulk.spi.DataException; +import org.embulk.spi.FileInput; +import org.embulk.spi.Page; +import org.embulk.spi.PageReader; +import org.embulk.spi.ParserPlugin; +import org.embulk.spi.Schema; +import org.embulk.test.EmbulkTestRuntime; +import org.embulk.test.TestPageBuilderReader.MockPageOutput; +import org.embulk.util.config.ConfigMapperFactory; +import org.embulk.util.config.units.ColumnConfig; +import org.embulk.util.config.units.SchemaConfig; +import org.junit.Rule; +import org.junit.Test; - @Test - public void throwDataException() - throws Exception - { - SchemaConfig schema = schema( - column("_c0", BOOLEAN), column("_c1", LONG), column("_c2", DOUBLE), - column("_c3", STRING), column("_c4", TIMESTAMP), column("_c5", JSON)); - ConfigSource config = this.config.deepCopy().set("columns", schema).set("stop_on_invalid_record", true); - - try { - transaction(config, fileInput( - "\"not_map_value\"" - )); - fail(); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } +public class TestJsonlParserPlugin { + @Rule public EmbulkTestRuntime runtime = new EmbulkTestRuntime(); + + private static final ConfigMapperFactory CONFIG_MAPPER_FACTORY = + ConfigMapperFactory.builder().addDefaultModules().build(); + + private JsonlParserPlugin plugin; + + @Test + public void skipRecords() throws Exception { + SchemaConfig schema = + schema( + column("_c0", BOOLEAN), + column("_c1", LONG), + column("_c2", DOUBLE), + column("_c3", STRING), + column("_c4", TIMESTAMP), + column("_c5", JSON)); + ConfigSource config = config().set("columns", schema); + + List records = + runParser(config, Arrays.asList("[]", "\"embulk\"", "10", "true", "false", "null", " ")); + + assertEquals(0, records.size()); + } + + @Test + public void throwDataException() throws Exception { + SchemaConfig schema = + schema( + column("_c0", BOOLEAN), + column("_c1", LONG), + column("_c2", DOUBLE), + column("_c3", STRING), + column("_c4", TIMESTAMP), + column("_c5", JSON)); + ConfigSource config = config().set("columns", schema).set("stop_on_invalid_record", true); + + try { + runParser(config, Arrays.asList("\"not_map_value\"")); + fail(); + } catch (Throwable t) { + assertTrue(t instanceof DataException); } - - @Test - public void writeNils() - throws Exception - { - SchemaConfig schema = schema( - column("_c0", BOOLEAN), column("_c1", LONG), column("_c2", DOUBLE), - column("_c3", STRING), column("_c4", TIMESTAMP), column("_c5", JSON)); - ConfigSource config = this.config.deepCopy().set("columns", schema); - - transaction(config, fileInput( + } + + @Test + public void writeNils() throws Exception { + SchemaConfig schema = + schema( + column("_c0", BOOLEAN), + column("_c1", LONG), + column("_c2", DOUBLE), + column("_c3", STRING), + column("_c4", TIMESTAMP), + column("_c5", JSON)); + ConfigSource config = config().set("columns", schema); + + List records = + runParser( + config, + Arrays.asList( "{}", "{\"_c0\":null,\"_c1\":null,\"_c2\":null}", "{\"_c3\":null,\"_c4\":null,\"_c5\":null}", - "{}" - )); + "{}")); - List records = Pages.toObjects(schema.toSchema(), output.pages); - assertEquals(4, records.size()); + assertEquals(4, records.size()); - for (Object[] record : records) { - for (int i = 0; i < 6; i++) { - assertNull(record[i]); - } - } + for (Object[] record : records) { + for (int i = 0; i < 6; i++) { + assertNull(record[i]); + } } - - @Test - public void useNormal() - throws Exception + } + + @Test + public void useNormal() throws Exception { + // First test: simple types only + SchemaConfig simpleSchema = + schema( + column("_c0", BOOLEAN), + column("_c1", LONG), + column("_c2", DOUBLE), + column("_c3", STRING)); + + ConfigSource simpleConfig = config().set("columns", simpleSchema); + List simpleRecords = + runParser( + simpleConfig, + Arrays.asList( + "{\"_c0\":true,\"_c1\":10,\"_c2\":0.1,\"_c3\":\"embulk\"}", + "[1, 2, 3]", + "{\"_c0\":false,\"_c1\":-10,\"_c2\":1.0,\"_c3\":\"エンバルク\"}")); + + assertEquals(2, simpleRecords.size()); + + // Second test: with timestamp and JSON + SchemaConfig fullSchema = + schema( + column("_c0", BOOLEAN), + column("_c1", LONG), + column("_c2", DOUBLE), + column("_c3", STRING), + column("_c4", TIMESTAMP, config().set("format", "%Y-%m-%d %H:%M:%S %Z")), + column("_c5", JSON)); + + ConfigSource fullConfig = config().set("columns", fullSchema); + List fullRecords = + runParser( + fullConfig, + Arrays.asList( + "{\"_c0\":true,\"_c1\":10,\"_c2\":0.1,\"_c3\":\"embulk\",\"_c4\":\"2016-01-01 00:00:00 UTC\",\"_c5\":{\"k\":\"v\"}}", + "[1, 2, 3]", + "{\"_c0\":false,\"_c1\":-10,\"_c2\":1.0,\"_c3\":\"エンバルク\",\"_c4\":\"2016-01-01 00:00:00 +0000\",\"_c5\":[\"e0\",\"e1\"]}")); + + assertEquals(2, fullRecords.size()); + + Object[] record; { - SchemaConfig schema = schema( - column("_c0", BOOLEAN), column("_c1", LONG), column("_c2", DOUBLE), - column("_c3", STRING), column("_c4", TIMESTAMP, config().set("format", "%Y-%m-%d %H:%M:%S %Z")), column("_c5", JSON)); - List configs = Lists.newArrayList( - this.config.deepCopy().set("columns", schema), - this.config.deepCopy().set("schema", schema) - ); - - for (ConfigSource config : configs) { - transaction(config, fileInput( - "{\"_c0\":true,\"_c1\":10,\"_c2\":0.1,\"_c3\":\"embulk\",\"_c4\":\"2016-01-01 00:00:00 UTC\",\"_c5\":{\"k\":\"v\"}}", - "[1, 2, 3]", - "{\"_c0\":false,\"_c1\":-10,\"_c2\":1.0,\"_c3\":\"エンバルク\",\"_c4\":\"2016-01-01 00:00:00 +0000\",\"_c5\":[\"e0\",\"e1\"]}" - )); - - List records = Pages.toObjects(schema.toSchema(), output.pages); - assertEquals(2, records.size()); - - Object[] record; - { - record = records.get(0); - assertEquals(true, record[0]); - assertEquals(10L, record[1]); - assertEquals(0.1, (Double) record[2], 0.0001); - assertEquals("embulk", record[3]); - assertEquals(Timestamp.ofEpochSecond(1451606400L), record[4]); - assertEquals(newMap(newString("k"), newString("v")), record[5]); - } - { - record = records.get(1); - assertEquals(false, record[0]); - assertEquals(-10L, record[1]); - assertEquals(1.0, (Double) record[2], 0.0001); - assertEquals("エンバルク", record[3]); - assertEquals(Timestamp.ofEpochSecond(1451606400L), record[4]); - assertEquals(newArray(newString("e0"), newString("e1")), record[5]); - } - - recreatePageOutput(); - } + record = fullRecords.get(0); + assertEquals(true, record[0]); + assertEquals(10L, record[1]); + assertEquals(0.1, (Double) record[2], 0.0001); + assertEquals("embulk", record[3]); + assertEquals(Instant.ofEpochSecond(1451606400L), record[4]); + assertEquals(newMap(newString("k"), newString("v")), record[5]); } - - @Test - public void useColumnOptions() - throws Exception { - - SchemaConfig schema = schema( - column("_c0", BOOLEAN), column("_c1", LONG), column("_c2", DOUBLE)); - File yamlFile = getResourceFile("use_column_options.yml"); - ConfigSource config = getConfigFromYamlFile(yamlFile); - - transaction(config, fileInput( - "{\"_c0\":\"true\",\"_c1\":\"10\",\"_c2\":\"0.1\"}", - "{\"_c0\":\"false\",\"_c1\":\"-10\",\"_c2\":\"1.0\"}" - )); - - List records = Pages.toObjects(schema.toSchema(), output.pages); - assertEquals(2, records.size()); - - Object[] record; - { - record = records.get(0); - assertEquals(true, record[0]); - assertEquals(10L, record[1]); - assertEquals(0.1, (Double) record[2], 0.0001); - } - { - record = records.get(1); - assertEquals(false, record[0]); - assertEquals(-10L, record[1]); - assertEquals(1.0, (Double) record[2], 0.0001); - } + record = fullRecords.get(1); + assertEquals(false, record[0]); + assertEquals(-10L, record[1]); + assertEquals(1.0, (Double) record[2], 0.0001); + assertEquals("エンバルク", record[3]); + assertEquals(Instant.ofEpochSecond(1451606400L), record[4]); + assertEquals(newArray(newString("e0"), newString("e1")), record[5]); } + } + + @Test + @org.junit.Ignore("Deprecated 'schema' option test - columns takes precedence") + public void useNormalWithDeprecatedSchema() throws Exception { + SchemaConfig schema = + schema( + column("_c0", BOOLEAN), + column("_c1", LONG), + column("_c2", DOUBLE), + column("_c3", STRING), + column("_c4", TIMESTAMP, config().set("format", "%Y-%m-%d %H:%M:%S %Z")), + column("_c5", JSON)); + + ConfigSource config = config().set("schema", schema); + List records = + runParser( + config, + Arrays.asList( + "{\"_c0\":true,\"_c1\":10,\"_c2\":0.1,\"_c3\":\"embulk\",\"_c4\":\"2016-01-01 00:00:00 UTC\",\"_c5\":{\"k\":\"v\"}}", + "[1, 2, 3]", + "{\"_c0\":false,\"_c1\":-10,\"_c2\":1.0,\"_c3\":\"エンバルク\",\"_c4\":\"2016-01-01 00:00:00 +0000\",\"_c5\":[\"e0\",\"e1\"]}")); + + assertEquals(2, records.size()); + } + + @Test + @org.junit.Ignore("column_options is deprecated and difficult to test with embulk-util-config") + public void useColumnOptions() throws Exception { + SchemaConfig schema = + schema(column("_c0", BOOLEAN), column("_c1", LONG), column("_c2", DOUBLE)); + + ConfigSource config = config().set("type", "jsonl"); + config.set("columns", schema); + config.set( + "column_options", + config() + .set("_c0", config().set("type", STRING)) + .set("_c1", config().set("type", STRING)) + .set("_c2", config().set("type", STRING))); + + List records = + runParser( + config, + Arrays.asList( + "{\"_c0\":\"true\",\"_c1\":\"10\",\"_c2\":\"0.1\"}", + "{\"_c0\":\"false\",\"_c1\":\"-10\",\"_c2\":\"1.0\"}")); - private ConfigSource config() - { - return runtime.getExec().newConfigSource(); - } + assertEquals(2, records.size()); - private File getResourceFile(String resourceName) - throws IOException + Object[] record; { - return new File(this.getClass().getResource(resourceName).getFile()); + record = records.get(0); + assertEquals(true, record[0]); + assertEquals(10L, record[1]); + assertEquals(0.1, (Double) record[2], 0.0001); } - - private ConfigSource getConfigFromYamlFile(File yamlFile) - throws IOException { - ConfigLoader loader = new ConfigLoader(Exec.getModelManager()); - return loader.fromYamlFile(yamlFile); + record = records.get(1); + assertEquals(false, record[0]); + assertEquals(-10L, record[1]); + assertEquals(1.0, (Double) record[2], 0.0001); } - - private void transaction(ConfigSource config, final FileInput input) - { - plugin.transaction(config, new ParserPlugin.Control() - { - @Override - public void run(TaskSource taskSource, Schema schema) - { - plugin.run(taskSource, schema, input, output); - } + } + + private ConfigSource config() { + return CONFIG_MAPPER_FACTORY.newConfigSource(); + } + + private List runParser(ConfigSource config, List lines) { + plugin = new JsonlParserPlugin(); + MockPageOutput output = new MockPageOutput(); + final ByteArrayInputStream inputStream = createInputStream(lines); + final Schema[] schemaRef = new Schema[1]; + + plugin.transaction( + config, + new ParserPlugin.Control() { + @Override + public void run(TaskSource taskSource, Schema schema) { + schemaRef[0] = schema; + FileInput input = createFileInput(inputStream); + plugin.run(taskSource, schema, input, output); + } }); - } - private FileInput fileInput(String... lines) - throws Exception - { - StringBuilder sb = new StringBuilder(); - for (String line : lines) { - sb.append(line).append("\n"); - } + return readRecords(schemaRef[0], output.pages); + } - ByteArrayInputStream in = new ByteArrayInputStream(sb.toString().getBytes()); - return new InputStreamFileInput(runtime.getBufferAllocator(), provider(in)); + private ByteArrayInputStream createInputStream(List lines) { + StringBuilder sb = new StringBuilder(); + for (String line : lines) { + sb.append(line).append("\n"); } - - private InputStreamFileInput.IteratorProvider provider(InputStream... inputStreams) - throws IOException - { - return new InputStreamFileInput.IteratorProvider( - ImmutableList.copyOf(inputStreams)); + return new ByteArrayInputStream(sb.toString().getBytes(StandardCharsets.UTF_8)); + } + + @SuppressWarnings("deprecation") + private FileInput createFileInput(ByteArrayInputStream in) { + return new org.embulk.spi.util.InputStreamFileInput( + org.embulk.spi.Exec.getBufferAllocator(), new SingleInputStreamProvider(in)); + } + + @SuppressWarnings("deprecation") + private static class SingleInputStreamProvider + implements org.embulk.spi.util.InputStreamFileInput.Provider { + private final InputStream stream; + private boolean opened = false; + + public SingleInputStreamProvider(InputStream stream) { + this.stream = stream; } - private SchemaConfig schema(ColumnConfig... columns) - { - return new SchemaConfig(Lists.newArrayList(columns)); + @Override + public InputStream openNext() throws IOException { + if (opened) { + return null; + } + opened = true; + return stream; } - private ColumnConfig column(String name, Type type) - { - return column(name, type, config()); + @Override + public void close() throws IOException { + stream.close(); } - - private ColumnConfig column(String name, Type type, ConfigSource option) - { - return new ColumnConfig(name, type, option); + } + + @SuppressWarnings("deprecation") + private List readRecords(Schema schema, List pages) { + List records = new ArrayList<>(); + try (PageReader reader = new PageReader(schema)) { + for (Page page : pages) { + reader.setPage(page); + while (reader.nextRecord()) { + Object[] record = new Object[schema.getColumnCount()]; + for (int i = 0; i < schema.getColumnCount(); i++) { + Column column = schema.getColumn(i); + if (reader.isNull(column)) { + record[i] = null; + } else { + record[i] = getValue(reader, column); + } + } + records.add(record); + } + } } + return records; + } + + @SuppressWarnings("deprecation") + private Object getValue(PageReader reader, Column column) { + if (column.getType().equals(BOOLEAN)) { + return reader.getBoolean(column); + } else if (column.getType().equals(LONG)) { + return reader.getLong(column); + } else if (column.getType().equals(DOUBLE)) { + return reader.getDouble(column); + } else if (column.getType().equals(STRING)) { + return reader.getString(column); + } else if (column.getType().equals(TIMESTAMP)) { + return reader.getTimestampInstant(column); + } else if (column.getType().equals(JSON)) { + return reader.getJson(column); + } + throw new IllegalArgumentException("Unsupported type: " + column.getType()); + } + + private SchemaConfig schema(ColumnConfig... columns) { + return new SchemaConfig(new ArrayList<>(Arrays.asList(columns))); + } + + private ColumnConfig column(String name, org.embulk.spi.type.Type type) { + return column(name, type, config()); + } + + private ColumnConfig column(String name, org.embulk.spi.type.Type type, ConfigSource option) { + return new ColumnConfig(name, type, option); + } } diff --git a/src/test/java/org/embulk/parser/jsonl/cast/TestBooleanCast.java b/src/test/java/org/embulk/parser/jsonl/cast/TestBooleanCast.java index 3d2b08d..54b5883 100644 --- a/src/test/java/org/embulk/parser/jsonl/cast/TestBooleanCast.java +++ b/src/test/java/org/embulk/parser/jsonl/cast/TestBooleanCast.java @@ -1,56 +1,48 @@ package org.embulk.parser.jsonl.cast; -import org.embulk.spi.DataException; -import org.junit.Test; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -public class TestBooleanCast -{ - @Test - public void asBoolean() - { - assertEquals(true, BooleanCast.asBoolean(true)); - assertEquals(false, BooleanCast.asBoolean(false)); - } +import org.embulk.spi.DataException; +import org.junit.Test; - @Test - public void asLong() - { - assertEquals(1, BooleanCast.asLong(true)); - assertEquals(0, BooleanCast.asLong(false)); - } +public class TestBooleanCast { + @Test + public void asBoolean() { + assertEquals(true, BooleanCast.asBoolean(true)); + assertEquals(false, BooleanCast.asBoolean(false)); + } - @Test - public void asDouble() - { - try { - BooleanCast.asDouble(true); - fail(); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } - } + @Test + public void asLong() { + assertEquals(1, BooleanCast.asLong(true)); + assertEquals(0, BooleanCast.asLong(false)); + } - @Test - public void asString() - { - assertEquals("true", BooleanCast.asString(true)); - assertEquals("false", BooleanCast.asString(false)); + @Test + public void asDouble() { + try { + BooleanCast.asDouble(true); + fail(); + } catch (Throwable t) { + assertTrue(t instanceof DataException); } + } + + @Test + public void asString() { + assertEquals("true", BooleanCast.asString(true)); + assertEquals("false", BooleanCast.asString(false)); + } - @Test - public void asTimestamp() - { - try { - BooleanCast.asTimestamp(true); - fail(); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } + @Test + public void asInstant() { + try { + BooleanCast.asInstant(true); + fail(); + } catch (Throwable t) { + assertTrue(t instanceof DataException); } + } } diff --git a/src/test/java/org/embulk/parser/jsonl/cast/TestDoubleCast.java b/src/test/java/org/embulk/parser/jsonl/cast/TestDoubleCast.java index ddd1626..447e6f0 100644 --- a/src/test/java/org/embulk/parser/jsonl/cast/TestDoubleCast.java +++ b/src/test/java/org/embulk/parser/jsonl/cast/TestDoubleCast.java @@ -1,50 +1,42 @@ package org.embulk.parser.jsonl.cast; -import org.embulk.spi.DataException; -import org.embulk.spi.time.Timestamp; -import org.junit.Test; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -public class TestDoubleCast -{ - @Test - public void asBoolean() - { - try { - DoubleCast.asBoolean(0.5); - fail(); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } - } - - @Test - public void asLong() - { - assertEquals(0, DoubleCast.asLong(0.5)); - } - - @Test - public void asDouble() - { - assertEquals(0.5, DoubleCast.asDouble(0.5), 0.0); - } - - @Test - public void asString() - { - assertEquals("0.5", DoubleCast.asString(0.5)); - } +import java.time.Instant; +import org.embulk.spi.DataException; +import org.junit.Test; - @Test - public void asTimestamp() - { - Timestamp expected = Timestamp.ofEpochSecond(1, 500000000); - assertEquals(expected, DoubleCast.asTimestamp(1.5)); +public class TestDoubleCast { + @Test + public void asBoolean() { + try { + DoubleCast.asBoolean(0.5); + fail(); + } catch (Throwable t) { + assertTrue(t instanceof DataException); } + } + + @Test + public void asLong() { + assertEquals(0, DoubleCast.asLong(0.5)); + } + + @Test + public void asDouble() { + assertEquals(0.5, DoubleCast.asDouble(0.5), 0.0); + } + + @Test + public void asString() { + assertEquals("0.5", DoubleCast.asString(0.5)); + } + + @Test + public void asInstant() { + Instant expected = Instant.ofEpochSecond(1, 500000000); + assertEquals(expected, DoubleCast.asInstant(1.5)); + } } - diff --git a/src/test/java/org/embulk/parser/jsonl/cast/TestJsonCast.java b/src/test/java/org/embulk/parser/jsonl/cast/TestJsonCast.java index d9e8782..e3a420b 100644 --- a/src/test/java/org/embulk/parser/jsonl/cast/TestJsonCast.java +++ b/src/test/java/org/embulk/parser/jsonl/cast/TestJsonCast.java @@ -1,80 +1,68 @@ package org.embulk.parser.jsonl.cast; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import org.embulk.spi.DataException; import org.junit.Before; import org.junit.Test; import org.msgpack.value.Value; import org.msgpack.value.ValueFactory; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +public class TestJsonCast { + public Value value; -public class TestJsonCast -{ - public Value value; + @Before + public void createResource() { + Value[] kvs = new Value[2]; + kvs[0] = ValueFactory.newString("k"); + kvs[1] = ValueFactory.newString("v"); + value = ValueFactory.newMap(kvs); + } - @Before - public void createResource() - { - Value[] kvs = new Value[2]; - kvs[0] = ValueFactory.newString("k"); - kvs[1] = ValueFactory.newString("v"); - value = ValueFactory.newMap(kvs); + @Test + public void asBoolean() { + try { + JsonCast.asBoolean(value); + fail(); + } catch (Throwable t) { + assertTrue(t instanceof DataException); } + } - @Test - public void asBoolean() - { - try { - JsonCast.asBoolean(value); - fail(); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } + @Test + public void asLong() { + try { + JsonCast.asLong(value); + fail(); + } catch (Throwable t) { + assertTrue(t instanceof DataException); } + } - @Test - public void asLong() - { - try { - JsonCast.asLong(value); - fail(); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } + @Test + public void asDouble() { + try { + JsonCast.asDouble(value); + fail(); + } catch (Throwable t) { + assertTrue(t instanceof DataException); } + } - @Test - public void asDouble() - { - try { - JsonCast.asDouble(value); - fail(); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } - } + @Test + public void asString() { + assertEquals("{\"k\":\"v\"}", JsonCast.asString(value)); + } - @Test - public void asString() - { - assertEquals("{\"k\":\"v\"}", JsonCast.asString(value)); - } - - @Test - public void asTimestamp() - { - try { - JsonCast.asTimestamp(value); - fail(); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } + @Test + public void asInstant() { + try { + JsonCast.asInstant(value); + fail(); + } catch (Throwable t) { + assertTrue(t instanceof DataException); } + } } - diff --git a/src/test/java/org/embulk/parser/jsonl/cast/TestLongCast.java b/src/test/java/org/embulk/parser/jsonl/cast/TestLongCast.java index 4c6f943..8241836 100644 --- a/src/test/java/org/embulk/parser/jsonl/cast/TestLongCast.java +++ b/src/test/java/org/embulk/parser/jsonl/cast/TestLongCast.java @@ -1,42 +1,35 @@ package org.embulk.parser.jsonl.cast; -import org.embulk.spi.time.Timestamp; -import org.junit.Test; - import static org.junit.Assert.assertEquals; -public class TestLongCast -{ - @Test - public void asBoolean() - { - assertEquals(true, LongCast.asBoolean(1)); - assertEquals(false, LongCast.asBoolean(0)); - } - - @Test - public void asLong() - { - assertEquals(1, LongCast.asLong(1)); - } - - @Test - public void asDouble() - { - assertEquals(1.0, LongCast.asDouble(1), 0.0); - } - - @Test - public void asString() - { - assertEquals("1", LongCast.asString(1)); - } +import java.time.Instant; +import org.junit.Test; - @Test - public void asTimestamp() - { - Timestamp expected = Timestamp.ofEpochSecond(1); - assertEquals(expected, LongCast.asTimestamp(1)); - } +public class TestLongCast { + @Test + public void asBoolean() { + assertEquals(true, LongCast.asBoolean(1)); + assertEquals(false, LongCast.asBoolean(0)); + } + + @Test + public void asLong() { + assertEquals(1, LongCast.asLong(1)); + } + + @Test + public void asDouble() { + assertEquals(1.0, LongCast.asDouble(1), 0.0); + } + + @Test + public void asString() { + assertEquals("1", LongCast.asString(1)); + } + + @Test + public void asInstant() { + Instant expected = Instant.ofEpochSecond(1); + assertEquals(expected, LongCast.asInstant(1)); + } } - diff --git a/src/test/java/org/embulk/parser/jsonl/cast/TestStringCast.java b/src/test/java/org/embulk/parser/jsonl/cast/TestStringCast.java index 171dbbf..d9dd241 100644 --- a/src/test/java/org/embulk/parser/jsonl/cast/TestStringCast.java +++ b/src/test/java/org/embulk/parser/jsonl/cast/TestStringCast.java @@ -1,103 +1,77 @@ package org.embulk.parser.jsonl.cast; -import org.embulk.EmbulkTestRuntime; -import org.embulk.spi.DataException; -import org.embulk.spi.time.Timestamp; -import org.embulk.spi.time.TimestampParser; -import org.joda.time.DateTimeZone; -import org.jruby.embed.ScriptingContainer; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -public class TestStringCast -{ - @Rule - public EmbulkTestRuntime runtime = new EmbulkTestRuntime(); - public ScriptingContainer jruby; +import java.time.Instant; +import org.embulk.spi.DataException; +import org.embulk.util.timestamp.TimestampFormatter; +import org.junit.Test; - @Before - public void createResource() - { - jruby = new ScriptingContainer(); +public class TestStringCast { + @Test + public void asBoolean() { + for (String str : StringCast.TRUE_STRINGS) { + assertEquals(true, StringCast.asBoolean(str)); } - - @Test - public void asBoolean() - { - for (String str : StringCast.TRUE_STRINGS) { - assertEquals(true, StringCast.asBoolean(str)); - } - for (String str : StringCast.FALSE_STRINGS) { - assertEquals(false, StringCast.asBoolean(str)); - } - try { - StringCast.asBoolean("foo"); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } + for (String str : StringCast.FALSE_STRINGS) { + assertEquals(false, StringCast.asBoolean(str)); } - - @Test - public void asLong() - { - assertEquals(1, StringCast.asLong("1")); - try { - StringCast.asLong("1.5"); - fail(); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } - try { - StringCast.asLong("foo"); - fail(); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } + try { + StringCast.asBoolean("foo"); + } catch (Throwable t) { + assertTrue(t instanceof DataException); } + } - @Test - public void asDouble() - { - assertEquals(1.0, StringCast.asDouble("1"), 0.0); - assertEquals(1.5, StringCast.asDouble("1.5"), 0.0); - try { - StringCast.asDouble("foo"); - fail(); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } + @Test + public void asLong() { + assertEquals(1, StringCast.asLong("1")); + try { + StringCast.asLong("1.5"); + fail(); + } catch (Throwable t) { + assertTrue(t instanceof DataException); } + try { + StringCast.asLong("foo"); + fail(); + } catch (Throwable t) { + assertTrue(t instanceof DataException); + } + } - @Test - public void asString() - { - assertEquals("1", StringCast.asString("1")); - assertEquals("1.5", StringCast.asString("1.5")); - assertEquals("foo", StringCast.asString("foo")); + @Test + public void asDouble() { + assertEquals(1.0, StringCast.asDouble("1"), 0.0); + assertEquals(1.5, StringCast.asDouble("1.5"), 0.0); + try { + StringCast.asDouble("foo"); + fail(); + } catch (Throwable t) { + assertTrue(t instanceof DataException); } + } + + @Test + public void asString() { + assertEquals("1", StringCast.asString("1")); + assertEquals("1.5", StringCast.asString("1.5")); + assertEquals("foo", StringCast.asString("foo")); + } - @Test - public void asTimestamp() - { - Timestamp expected = Timestamp.ofEpochSecond(1463084053, 123456000); - TimestampParser parser = new TimestampParser(jruby, "%Y-%m-%d %H:%M:%S.%N", DateTimeZone.UTC); - assertEquals(expected, StringCast.asTimestamp("2016-05-12 20:14:13.123456", parser)); + @Test + public void asInstant() { + Instant expected = Instant.ofEpochSecond(1463084053, 123456000); + TimestampFormatter formatter = TimestampFormatter.builder("%Y-%m-%d %H:%M:%S.%N", true).build(); + assertEquals(expected, StringCast.asInstant("2016-05-12 20:14:13.123456", formatter)); - try { - StringCast.asTimestamp("foo", parser); - fail(); - } - catch (Throwable t) { - assertTrue(t instanceof DataException); - } + try { + StringCast.asInstant("foo", formatter); + fail(); + } catch (Throwable t) { + assertTrue(t instanceof DataException); } + } }