Error running ./snowplow-emr-etl-runner run -c config/config.yml -r config/iglu_resolver.json


#1

Hi all,

I’m trying to setup Snowplow on my Mac OSX machine but I’m running in to a problem when I try to run the following code:

./snowplow-emr-etl-runner run -c config/config.yml -r config/iglu_resolver.json

I’m ending up with the following error. Does anyone know how we can solve this?

WARNING: An illegal reflective access operation has occurred
WARNING: Illegal reflective access by org.jruby.util.io.FilenoUtil (file:/var/folders/54/szqpyj8j3vgccct0qh9s2f300000gn/T/jruby14489241855663513793extract/jruby-core-9.1.7.0-complete.jar) to method sun.nio.ch.SelChImpl.getFD()
WARNING: Please consider reporting this to the maintainers of org.jruby.util.io.FilenoUtil
WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
WARNING: All illegal access operations will be denied in a future release
F, [2018-02-21T11:47:34.462000 #1801] FATAL -- : 

ReturnContractError (Contract violation for return value:
        Expected: #<Contracts::Maybe:0x147203c2 @vals=[{:aws=>{:access_key_id=>String, :secret_access_key=>String, :s3=>{:region=>String, :buckets=>{:assets=>String, :jsonpath_assets=>#<Contracts::Maybe:0x11678469 @vals=[String, nil]>, :log=>String, :raw=>{:in=>#<Contracts::CollectionOf:0x278cbf5a @contract=String, @collection_class=Array>, :processing=>String, :archive=>String}, :enriched=>{:good=>String, :bad=>String, :errors=>#<Contracts::Maybe:0x73e63b4d @vals=[String, nil]>, :archive=>#<Contracts::Maybe:0x8aafd70 @vals=[String, nil]>}, :shredded=>{:good=>String, :bad=>String, :errors=>#<Contracts::Maybe:0x3e5ed0d4 @vals=[String, nil]>, :archive=>#<Contracts::Maybe:0x61c0fb12 @vals=[String, nil]>}}}, :emr=>{:ami_version=>String, :region=>String, :jobflow_role=>String, :service_role=>String, :placement=>#<Contracts::Maybe:0x16a15261 @vals=[String, nil]>, :ec2_subnet_id=>#<Contracts::Maybe:0x5d8112e6 @vals=[String, nil]>, :ec2_key_name=>String, :bootstrap=>#<Contracts::Maybe:0x68cc6319 @vals=[#<Contracts::CollectionOf:0x1a507ea0 @contract=String, @collection_class=Array>, nil]>, :software=>{:hbase=>#<Contracts::Maybe:0x59700156 @vals=[String, nil]>, :lingual=>#<Contracts::Maybe:0x7359f3ac @vals=[String, nil]>}, :jobflow=>{:job_name=>String, :master_instance_type=>String, :core_instance_count=>Contracts::Num, :core_instance_type=>String, :core_instance_ebs=>#<Contracts::Maybe:0x1adec8d @vals=[{:volume_size=>#<Proc:0x36def543@uri:classloader:/emr-etl-runner/lib/snowplow-emr-etl-runner/contracts.rb:28 (lambda)>, :volume_type=>#<Proc:0x2e07dc23@uri:classloader:/emr-etl-runner/lib/snowplow-emr-etl-runner/contracts.rb:27 (lambda)>, :volume_iops=>#<Contracts::Maybe:0x7859fc7 @vals=[#<Proc:0x36def543@uri:classloader:/emr-etl-runner/lib/snowplow-emr-etl-runner/contracts.rb:28 (lambda)>, nil]>, :ebs_optimized=>#<Contracts::Maybe:0x10ab976b @vals=[Contracts::Bool, nil]>}, nil]>, :task_instance_count=>Contracts::Num, :task_instance_type=>String, :task_instance_bid=>#<Contracts::Maybe:0x577f81a0 @vals=[Contracts::Num, nil]>}, :additional_info=>#<Contracts::Maybe:0x1256925b @vals=[String, nil]>, :bootstrap_failure_tries=>Contracts::Num, :configuration=>#<Contracts::Maybe:0xeb09112 @vals=[#<Contracts::HashOf:0x2bc52b08 @key=Symbol, @value=#<Contracts::HashOf:0x6b578779 @key=Symbol, @value=String>>, nil]>}}, :collectors=>{:format=>String}, :enrich=>{:versions=>{:spark_enrich=>String}, :continue_on_unexpected_error=>Contracts::Bool, :output_compression=>#<Proc:0x4e797a39@uri:classloader:/emr-etl-runner/lib/snowplow-emr-etl-runner/contracts.rb:26 (lambda)>}, :storage=>{:versions=>{:rdb_shredder=>String, :hadoop_elasticsearch=>String, :rdb_loader=>String}}, :monitoring=>{:tags=>#<Contracts::HashOf:0x112b3298 @key=Symbol, @value=String>, :logging=>{:level=>String}, :snowplow=>#<Contracts::Maybe:0x36696821 @vals=[{:method=>String, :collector=>String, :app_id=>String}, nil]>}}, nil]>,
        Actual: {:ws=>{:access_key_id=>"AKIAJDMZ7FIFQ4C3V3LQ3", :secret_access_key=>"krXTd0FZgz1vrmvsXFfm0c0PLJQuyAk4MEDEOGqK", :s3=>{:region=>"eu-west-1", :buckets=>{:assets=>"s3://snowplow-hosted-assets", :jsonpath_assets=>nil, :log=>"s3://simoahava-snowplow-log", :raw=>{:in=>["s3://elasticbeanstalk-eu-west-1-878816122560/resources/environments/logs/publish/e-wgeekfavfc"], :processing=>"s3://monaerk-snowplow-data/processing", :archive=>"s3://monaerk-snowplow-data/archive/raw"}, :enriched=>{:good=>"s3://monaerk-snowplow-data/enriched/good", :bad=>"s3://monaerk-snowplow-data/enriched/bad", :errors=>nil, :archive=>"s3://monaerk-snowplow-data/archive/enriched"}, :shredded=>{:good=>"s3://monaerk-snowplow-data/shredded/good", :bad=>"s3://monaerk-snowplow-data/shredded/bad", :errors=>nil, :archive=>"s3://monaerk-snowplow-data/archive/shredded"}}}, :emr=>{:ami_version=>"5.9.0", :region=>"eu-west-1", :jobflow_role=>"EMR_EC2_DefaultRole", :service_role=>"EMR_DefaultRole", :placement=>nil, :ec2_subnet_id=>"subnet-243dc942", :ec2_key_name=>"monaerk", :bootstrap=>[], :software=>{:hbase=>nil, :lingual=>nil}, :jobflow=>{:job_name=>"Snowplow ETL", :master_instance_type=>"m1.medium", :core_instance_count=>2, :core_instance_type=>"m1.medium", :core_instance_ebs=>{:volume_size=>100, :volume_type=>"gp2", :volume_iops=>400, :ebs_optimized=>false}, :task_instance_count=>0, :task_instance_type=>"m1.medium", :task_instance_bid=>0.015}, :bootstrap_failure_tries=>3, :configuration=>{:"yarn-site"=>{:"yarn.resourcemanager.am.max-attempts"=>"1"}, :spark=>{:maximizeResourceAllocation=>"true"}}, :additional_info=>nil}}, :collectors=>{:format=>"clj-tomcat"}, :enrich=>{:versions=>{:spark_enrich=>"1.12.0"}, :continue_on_unexpected_error=>false, :output_compression=>"NONE"}, :storage=>{:versions=>{:rdb_loader=>"0.14.0", :rdb_shredder=>"0.13.0", :hadoop_elasticsearch=>"0.1.0"}}, :monitoring=>{:tags=>{}, :logging=>{:level=>"DEBUG"}}}
        Value guarded in: Snowplow::EmrEtlRunner::Cli::load_config
        With Contract: Maybe, String, Bool => Maybe
        At: uri:classloader:/emr-etl-runner/lib/snowplow-emr-etl-runner/cli.rb:202 ):
    uri:classloader:/gems/contracts-0.11.0/lib/contracts.rb:45:in `block in Contract'
    uri:classloader:/gems/contracts-0.11.0/lib/contracts.rb:154:in `failure_callback'
    uri:classloader:/gems/contracts-0.11.0/lib/contracts/call_with.rb:80:in `call_with'
    uri:classloader:/gems/contracts-0.11.0/lib/contracts/method_handler.rb:138:in `block in redefine_method'
    uri:classloader:/emr-etl-runner/lib/snowplow-emr-etl-runner/cli.rb:191:in `process_options'
    uri:classloader:/emr-etl-runner/lib/snowplow-emr-etl-runner/cli.rb:167:in `get_args_config_enrichments_resolver'
    uri:classloader:/gems/contracts-0.11.0/lib/contracts/method_reference.rb:43:in `send_to'
    uri:classloader:/gems/contracts-0.11.0/lib/contracts/call_with.rb:76:in `call_with'
    uri:classloader:/gems/contracts-0.11.0/lib/contracts/method_handler.rb:138:in `block in redefine_method'
    uri:classloader:/emr-etl-runner/bin/snowplow-emr-etl-runner:37:in `<main>'
    org/jruby/RubyKernel.java:979:in `load'
    uri:classloader:/META-INF/main.rb:1:in `<main>'
    org/jruby/RubyKernel.java:961:in `require'
    uri:classloader:/META-INF/main.rb:1:in `(root)'
    uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/rubygems/core_ext/kernel_require.rb:1:in `<main>'

Many thanks for your replies.


#2

Hi @BrentStevens,

The error message means that your config.yml can’t be validated. It can be a wrong indentation, wrong or missing non-optional values and wrong version of the file.

Which version of Snowplow do you use? Could you share your configuration file without credentials? You might also need to reset the actual ones since the error message includes them.

Regards,