diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index 7ef4a2f..3f86f84 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -1,306 +1,306 @@ -#Tue Jan 13 16:30:02 IST 2015 -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert -org.eclipse.jdt.core.formatter.brace_position_for_block=end_of_line -org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert +eclipse.preferences.version=1 encoding/src/main/java=UTF-8 -org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert -org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line -org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert -org.eclipse.jdt.core.formatter.alignment_for_assignment=0 -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert -org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources=insert -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 encoding/src/main/resources=UTF-8 -org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert -org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false -org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false -org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert -org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert -org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert -org.eclipse.jdt.core.formatter.blank_lines_after_imports=1 -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation=0 -org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert -org.eclipse.jdt.core.formatter.comment.format_source_code=true -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert -org.eclipse.jdt.core.compiler.debug.localVariable=generate -org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true -org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package=insert -org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1 -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try=insert -org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert -org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert -org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column=true -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert -org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=1 -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=16 -org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1 +encoding/src/test/java=UTF-8 encoding/src/test/resources=UTF-8 -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert +org.eclipse.jdt.core.codeComplete.argumentPrefixes= +org.eclipse.jdt.core.codeComplete.argumentSuffixes= +org.eclipse.jdt.core.codeComplete.fieldPrefixes= +org.eclipse.jdt.core.codeComplete.fieldSuffixes= +org.eclipse.jdt.core.codeComplete.localPrefixes= +org.eclipse.jdt.core.codeComplete.localSuffixes= +org.eclipse.jdt.core.codeComplete.staticFieldPrefixes= +org.eclipse.jdt.core.codeComplete.staticFieldSuffixes= +org.eclipse.jdt.core.codeComplete.staticFinalFieldPrefixes= +org.eclipse.jdt.core.codeComplete.staticFinalFieldSuffixes= +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line -org.eclipse.jdt.core.formatter.comment.format_header=false -org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert -org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert +org.eclipse.jdt.core.compiler.compliance=1.7 +org.eclipse.jdt.core.compiler.debug.lineNumber=generate +org.eclipse.jdt.core.compiler.debug.localVariable=generate +org.eclipse.jdt.core.compiler.debug.sourceFile=generate +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.source=1.7 +org.eclipse.jdt.core.formatter.align_type_members_on_columns=false +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation=0 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=16 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=16 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=16 +org.eclipse.jdt.core.formatter.alignment_for_assignment=0 +org.eclipse.jdt.core.formatter.alignment_for_binary_expression=16 +org.eclipse.jdt.core.formatter.alignment_for_compact_if=16 +org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80 +org.eclipse.jdt.core.formatter.alignment_for_enum_constants=0 +org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16 +org.eclipse.jdt.core.formatter.alignment_for_method_declaration=0 +org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16 +org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_resources_in_try=80 +org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=16 org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration=16 -org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration=16 -org.eclipse.jdt.core.formatter.comment.line_length=80 -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert -org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert -org.eclipse.jdt.core.formatter.insert_space_after_binary_operator=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block=insert -org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert -org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=end_of_line -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert -org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert -org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=true -org.eclipse.jdt.core.formatter.continuation_indentation=2 -org.eclipse.jdt.core.codeComplete.fieldSuffixes= -org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field=insert -org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert -org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=16 -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert -org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert +org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch=16 +org.eclipse.jdt.core.formatter.blank_lines_after_imports=1 +org.eclipse.jdt.core.formatter.blank_lines_after_package=1 +org.eclipse.jdt.core.formatter.blank_lines_before_field=0 +org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0 +org.eclipse.jdt.core.formatter.blank_lines_before_imports=1 +org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1 org.eclipse.jdt.core.formatter.blank_lines_before_method=1 -org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true -org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16 +org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=1 +org.eclipse.jdt.core.formatter.blank_lines_before_package=0 +org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1 org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations=1 -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert -org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert +org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_block=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=end_of_line org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration=end_of_line -org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=16 -org.eclipse.jdt.core.formatter.join_lines_in_comments=true -org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true -org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert -org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0 +org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line +org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false +org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false org.eclipse.jdt.core.formatter.comment.format_block_comments=true -org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert -org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16 -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert -org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch=16 +org.eclipse.jdt.core.formatter.comment.format_header=false org.eclipse.jdt.core.formatter.comment.format_html=true -org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert -org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16 -org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert -org.eclipse.jdt.core.compiler.problem.enumIdentifier=error -org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert -org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert -org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert -org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=end_of_line -org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16 -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert +org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=true +org.eclipse.jdt.core.formatter.comment.format_line_comments=true +org.eclipse.jdt.core.formatter.comment.format_source_code=true +org.eclipse.jdt.core.formatter.comment.indent_parameter_description=true +org.eclipse.jdt.core.formatter.comment.indent_root_tags=true +org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert +org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=insert +org.eclipse.jdt.core.formatter.comment.line_length=80 +org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries=true org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries=true -org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments=false +org.eclipse.jdt.core.formatter.compact_else_if=true +org.eclipse.jdt.core.formatter.continuation_indentation=2 +org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2 +org.eclipse.jdt.core.formatter.disabling_tag=@formatter\:off +org.eclipse.jdt.core.formatter.enabling_tag=@formatter\:on +org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=false +org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column=true +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header=true -org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=insert -org.eclipse.jdt.core.codeComplete.staticFinalFieldSuffixes= -org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16 -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert -org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true +org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true +org.eclipse.jdt.core.formatter.indent_empty_lines=false +org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true +org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true +org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true +org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=false +org.eclipse.jdt.core.formatter.indentation.size=4 +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_label=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration=insert +org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block=insert +org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant=insert +org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration=insert +org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body=insert +org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert +org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16 -org.eclipse.jdt.core.codeComplete.fieldPrefixes= -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert -org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line -org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch=true +org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_binary_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert -org.eclipse.jdt.core.codeComplete.staticFieldSuffixes= -org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert -org.eclipse.jdt.core.formatter.alignment_for_enum_constants=0 -org.eclipse.jdt.core.formatter.alignment_for_method_declaration=0 -org.eclipse.jdt.core.formatter.tabulation.char=space -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert -org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert -org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=false -org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false -org.eclipse.jdt.core.formatter.compact_else_if=true -org.eclipse.jdt.core.codeComplete.localPrefixes= -org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert -org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert -org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=end_of_line -org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false -org.eclipse.jdt.core.formatter.disabling_tag=@formatter\:off -org.eclipse.jdt.core.formatter.align_type_members_on_columns=false -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type=insert -org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert +org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources=insert +org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert +org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_binary_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation=do not insert -org.eclipse.jdt.core.formatter.join_wrapped_lines=true -org.eclipse.jdt.core.codeComplete.localSuffixes= -org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional=insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert -org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true -org.eclipse.jdt.core.formatter.use_on_off_tags=false -org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16 -org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false -org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert -org.eclipse.jdt.core.formatter.comment.format_line_comments=true -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert -org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert -org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert -org.eclipse.jdt.core.formatter.comment.indent_parameter_description=true -org.eclipse.jdt.core.formatter.indentation.size=4 -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for=insert -org.eclipse.jdt.core.compiler.debug.sourceFile=generate +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert org.eclipse.jdt.core.formatter.insert_space_before_ellipsis=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert -org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16 -org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert -org.eclipse.jdt.core.formatter.comment.indent_root_tags=true +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant=insert -org.eclipse.jdt.core.formatter.alignment_for_binary_expression=16 -org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert -encoding/src/test/java=UTF-8 -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_binary_operator=insert -org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=false -org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true -org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert -org.eclipse.jdt.core.formatter.blank_lines_before_package=0 org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration=insert -org.eclipse.jdt.core.formatter.indent_empty_lines=false -org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=16 -org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested=true +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert -org.eclipse.jdt.core.codeComplete.staticFinalFieldPrefixes= -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert -org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=end_of_line -org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0 -org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert -org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80 -org.eclipse.jdt.core.codeComplete.argumentPrefixes= -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert -org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration=insert -org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=false -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant=insert -org.eclipse.jdt.core.codeComplete.argumentSuffixes= -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for=insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert -org.eclipse.jdt.core.formatter.alignment_for_resources_in_try=80 -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert -org.eclipse.jdt.core.codeComplete.staticFieldPrefixes= +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert -org.eclipse.jdt.core.compiler.debug.lineNumber=generate -org.eclipse.jdt.core.formatter.blank_lines_before_imports=1 -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert -org.eclipse.jdt.core.formatter.alignment_for_compact_if=16 -org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled -org.eclipse.jdt.core.formatter.insert_new_line_after_label=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert +org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert +org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert +org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert +org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources=do not insert org.eclipse.jdt.core.formatter.insert_space_before_unary_operator=do not insert -org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments=false -org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try=do not insert -org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2 +org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.join_lines_in_comments=true +org.eclipse.jdt.core.formatter.join_wrapped_lines=true +org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false +org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false +org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false +org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false org.eclipse.jdt.core.formatter.lineSplit=120 -org.eclipse.jdt.core.compiler.source=1.7 -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body=insert +org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false +org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false +org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0 +org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1 +org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true +org.eclipse.jdt.core.formatter.tabulation.char=space org.eclipse.jdt.core.formatter.tabulation.size=4 -org.eclipse.jdt.core.compiler.problem.assertIdentifier=error -org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert -org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1 -org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries=true -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert -org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert -org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert -eclipse.preferences.version=1 -org.eclipse.jdt.core.formatter.enabling_tag=@formatter\:on -org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert -org.eclipse.jdt.core.compiler.compliance=1.7 -org.eclipse.jdt.core.formatter.blank_lines_after_package=1 -org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16 -org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert -org.eclipse.jdt.core.formatter.blank_lines_before_field=0 +org.eclipse.jdt.core.formatter.use_on_off_tags=false +org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=false +org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true +org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch=true +org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested=true diff --git a/README.md b/README.md index 131308c..ab92da3 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,14 @@ Takipi Storage ============== +Thanks to Moovel for developing this version supporting s3: https://github.com/moovel/takipi-storage/tree/s3-storage +And to Atlassian for adding some extra features + Build and run: - clone the repo - `cd takipi-storage` - `mvn compile package` +- edit settings.yml to contain bucket, key, secret to access your s3 bucket - `java -jar target/takipi-storage-1.7.0.jar server settings.yml` Deploy: diff --git a/docker/Dockerfile b/docker/Dockerfile index 9199487..31c28f2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,21 +1,35 @@ -# Run me with docker run -v :/opt/takipi-storage/storage -p :$STORAGE_PORT -# Logs are written to /log folder and both should be persistent outside of the running container +FROM openjdk:8-jre-slim +LABEL maintainer="support@overops.com" -FROM java:8 -MAINTAINER Chen harel "https://github.com/chook" +ARG APP_VERSION=latest -ENV VERSION 1.7.0 -ENV TAR_FILENAME takipi-storage-$VERSION.tar.gz -ENV JAR_FILENAME takipi-storage-$VERSION.jar -ENV STORAGE_PORT 8080 +# install curl +RUN apt-get update; apt-get install -y curl -RUN wget https://s3.amazonaws.com/app-takipi-com/deploy/takipi-storage/$TAR_FILENAME -RUN tar zxvf $TAR_FILENAME -C /tmp && \ - mkdir -p /opt/takipi-storage/lib && \ - cp /tmp/takipi-storage/lib/$JAR_FILENAME /opt/takipi-storage/lib -ADD settings.yml /opt/takipi-storage +# rootless +RUN groupadd --gid 1000 overops +RUN adduser --home /opt/takipi-storage --uid 1000 --gid 1000 overops +USER 1000:1000 -EXPOSE $STORAGE_PORT +# install into the /opt directory +WORKDIR /opt + +# download and install the storage server +RUN curl -sL http://app-takipi-com.s3.amazonaws.com/deployx/s3/deploy/takipi-storage/takipi-storage-${APP_VERSION}.tar.gz | tar -xvzf - + +RUN mkdir /opt/takipi-storage/private +COPY --chown=1000:1000 "./private/settings.yaml" "/opt/takipi-storage/private/settings.yaml" + +# use mount to make settings.yaml available +VOLUME ["/opt/takipi-storage/private"] WORKDIR /opt/takipi-storage -CMD java -jar /opt/takipi-storage/lib/$JAR_FILENAME server settings.yml + +# copy the run script +COPY --chown=1000:1000 "./scripts/run.sh" "./run.sh" +RUN chmod +x run.sh + +EXPOSE 8080 8081 + +# run the service, printing logs to stdout +CMD ["./run.sh"] diff --git a/docker/Jenkinsfile b/docker/Jenkinsfile new file mode 100644 index 0000000..d767ebb --- /dev/null +++ b/docker/Jenkinsfile @@ -0,0 +1,72 @@ +def imageName = 'docker/overops-storage-server-s3' +def dockerHubImage = 'overops/storage-server-s3' + +pipeline { + + environment { + registryCred = 'container-registry-build-guy' + dockerhubCred = 'docker-hub' + gitCred = 'build-guy' + } + + parameters { + string(name: 'VERSION', defaultValue: 'latest', description:'Application version') + string(name: 'TAG', defaultValue: 'latest', description:'Image Tag to be used') + booleanParam(name: 'PUBLISH_TO_DOCKERHUB', defaultValue: false, description:'Flag to publish to docker-hub') + } + + agent any + stages { + stage('Cloning Git') { + steps { + git([url: 'https://github.com/takipi/takipi-storage', branch: 's3-storage', credentialsId: gitCred]) + } + } + + stage('Build Docker Image') { + steps { + dir('docker') { + script { + if (params.PUBLISH_TO_DOCKERHUB) { + imageName = dockerHubImage + } + + dockerOptions = ('--label=storage-server-s3-pipeline --build-arg APP_VERSION=' + params.VERSION + ' .') + dockerImage = docker.build(imageName, dockerOptions) + } + } + } + } + + stage('Publish Docker Image') { + steps { + script { + if (params.PUBLISH_TO_DOCKERHUB) { + reg = '' + cred = dockerhubCred + } else { + reg = env.LOCAL_DOCKER_REGISTRY_URL + cred = registryCred + } + + docker.withRegistry(reg, cred) { + dockerImage.push() + + if (params.TAG != 'latest') { + dockerImage.push(params.TAG) + } + } + } + } + } + + stage('Cleanup') { + steps { + script { + sh(script:"docker rmi -f \$(docker images -f label=storage-server-s3-pipeline -q)") + } + cleanWs() + } + } + } +} diff --git a/docker/README.md b/docker/README.md index e21d6ad..31e3826 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,6 +1,19 @@ -takipi-storage on docker -======================== +# Deploy the Storage Server - S3 (hybrid installations, AWS S3) -When running takipi-storage inside docker, make sure to persist the storage folder outside (using volumes). +For hybrid installations, the Storage Server can be installed in your cluster. -Logs are placed inside the storage folder +This Storage Server [Dockerfile](Dockerfile) is based on the [Installing the Storage Server on AWS S3](https://doc.overops.com/docs/installing-the-storage-server-on-aws-s3) guide, with some minor modifications. + +For complete instructions on performing a hybrid installation, refer to the [Hybrid Installation on Linux](https://doc.overops.com/docs/linux-hybrid-installation) guide. + +The file `settings.yaml` must be mounted into the `/opt/takipi-storage/private` directory to run this container. An example [settings.yaml](private/settings.yaml) can be found in this repo. + +## Quick Start + +This image is on Docker Hub: [overops/storage-server-s3](https://hub.docker.com/r/overops/storage-server-s3) + +### Docker Quick Start + +```console +docker run -d -p 8080:8080 --mount type=bind,source="$(pwd)"/private,target=/opt/takipi-storage/private overops/storage-server-s3 +``` \ No newline at end of file diff --git a/docker/private/settings.yaml b/docker/private/settings.yaml new file mode 100644 index 0000000..1d0df4a --- /dev/null +++ b/docker/private/settings.yaml @@ -0,0 +1,45 @@ +enableCors: true +corsOrigins: "*" + +# If using attaching IAM Role to instance leave accessKey and secretKey empty. +s3Fs: + bucket: + pathPrefix: + credentials: + accessKey: + secretKey: + +multifetch: + # maximum number of threads for concurrent multi-fetch. Set to zero to disable concurrent fetching. + concurrencyLevel: 100 + # Recommended size >= 16777216. Set to zero to disable caching. + maxCacheSize: 67108864 + enableCacheLogger: false + maxBatchSize: 4194304 + +server: + gzip: + includedMethods: [POST, GET] + applicationConnectors: + - type: http + port: 8080 +# https support +# - type: https +# port: 8443 +# keyStorePath: example.keystore +# keyStorePassword: example +# validateCerts: false + adminConnectors: + - type: http + port: 8081 + +# Logging settings. +logging: + # The default level of all loggers. Can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL. + level: INFO + # Logger-specific levels. + loggers: + com.takipi: DEBUG + + appenders: + - type: console diff --git a/docker/scripts/run.sh b/docker/scripts/run.sh new file mode 100644 index 0000000..7e14e5b --- /dev/null +++ b/docker/scripts/run.sh @@ -0,0 +1,3 @@ +#!/bin/bash +java -jar /opt/takipi-storage/lib/takipi-storage.jar server /opt/takipi-storage/private/settings.yaml &> /opt/takipi-storage/log/takipi-storage.log & +/usr/bin/tail -f /opt/takipi-storage/log/takipi-storage.log diff --git a/docker/settings.yml b/docker/settings.yml deleted file mode 100644 index 4dd3045..0000000 --- a/docker/settings.yml +++ /dev/null @@ -1,27 +0,0 @@ -folderPath: /opt/takipi-storage/storage -maxUsedStoragePercentage: 0.95 -enableCors: true -corsOrigins: "*" - -server: - applicationConnectors: - - type: http - port: 8080 - adminConnectors: - - type: http - port: 8081 - requestLog: - appenders: - - type: file - currentLogFilename: ./storage/log/access.log - archivedLogFilenamePattern: ./storage/log/access.%d.log.gz - archivedFileCount: 14 -logging: - level: INFO - loggers: - com.takipi: DEBUG - appenders: - - type: file - currentLogFilename: ./storage/log/takipi-storage.log - archivedLogFilenamePattern: ./storage/log/takipi-storage.%d.log.gz - archivedFileCount: 14 diff --git a/pom.xml b/pom.xml index cdbc0b0..a2bb4e1 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ UTF-8 UTF-8 - 1.2.2 + 0.7.1 0.7 @@ -30,6 +30,16 @@ junit 4.12 + + com.amazonaws + aws-java-sdk-s3 + 1.11.13 + + + com.google.guava + guava + 17.0 + @@ -98,7 +108,7 @@ org.codehaus.mojo findbugs-maven-plugin - 3.0.4-SNAPSHOT + 3.0.4 diff --git a/settings.yml b/settings.yml index 0157207..5d6533c 100644 --- a/settings.yml +++ b/settings.yml @@ -1,8 +1,31 @@ -folderPath: /opt/takipi-storage/storage -maxUsedStoragePercentage: 0.95 enableCors: true corsOrigins: "*" +# If using attaching IAM Role to instance leave accessKey and secretKey empty +s3Fs: + bucket: + pathPrefix: + credentials: + accessKey: + secretKey: + +multifetch: + + # maximum number of threads for concurrent multi-fetch. Set to zero to disable concurrent fetching. + concurrencyLevel: 100 + + # Recommended size >= 16777216. Set to zero to disable caching. + maxCacheSize: 67108864 + + enableCacheLogger: false + + maxBatchSize: 4194304 + +#folderFs: +# folderPath: /opt/takipi-storage/storage +# maxUsedStoragePercentage: 0.95 + + server: gzip: includedMethods: [POST, GET] @@ -11,9 +34,9 @@ server: applicationConnectors: - type: http port: 8080 -# this requires the alpn-boot library on the JVM's boot classpath -# - type: spdy3 -# port: 8445 +# https support +# - type: https +# port: 8443 # keyStorePath: example.keystore # keyStorePassword: example # validateCerts: false diff --git a/src/main/java/com/takipi/oss/storage/TakipiStorageConfiguration.java b/src/main/java/com/takipi/oss/storage/TakipiStorageConfiguration.java index 8693630..d026a29 100644 --- a/src/main/java/com/takipi/oss/storage/TakipiStorageConfiguration.java +++ b/src/main/java/com/takipi/oss/storage/TakipiStorageConfiguration.java @@ -1,7 +1,9 @@ package com.takipi.oss.storage; +import javax.validation.Valid; import javax.validation.constraints.Max; import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; import org.hibernate.validator.constraints.NotEmpty; @@ -10,13 +12,164 @@ import io.dropwizard.Configuration; public class TakipiStorageConfiguration extends Configuration { - @NotEmpty - private String folderPath; - @Min(0) - @Max(1) - private double maxUsedStoragePercentage = 0.9; + @Valid + @JsonProperty + private FolderFs folderFs; + + public static class FolderFs { + @NotEmpty + private String folderPath; + + @Min(0) + @Max(1) + private double maxUsedStoragePercentage = 0.9; + + @JsonProperty + public String getFolderPath() { + return folderPath; + } + + @JsonProperty + public void setFolderPath(String folderPath) { + this.folderPath = folderPath; + } + + @JsonProperty + public double getMaxUsedStoragePercentage() { + return maxUsedStoragePercentage; + } + + @JsonProperty + public void setMaxUsedStoragePercentage(double maxUsedStoragePercentage) { + this.maxUsedStoragePercentage = maxUsedStoragePercentage; + } + } + @Valid + @JsonProperty + private S3Fs s3Fs; + + public static class S3Fs { + + @NotEmpty + private String bucket; + + private String pathPrefix; + + @NotNull + @Valid + private Credentials credentials; + + public static class Credentials { + + private String accessKey; + + private String secretKey; + + @JsonProperty + public String getAccessKey() { + return TakipiStorageConfigurationEnvResolver.resolveEnv(accessKey); + } + + @JsonProperty + public void setAccessKey(String accessKey) { + this.accessKey = accessKey; + } + + @JsonProperty + public String getSecretKey() { + return TakipiStorageConfigurationEnvResolver.resolveEnv(secretKey); + } + + @JsonProperty + public void setSecretKey(String secretKey) { + this.secretKey = secretKey; + } + } + + @JsonProperty + public String getBucket() { + return TakipiStorageConfigurationEnvResolver.resolveEnv(bucket); + } + + @JsonProperty + public void setBucket(String bucket) { + this.bucket = bucket; + } + + @JsonProperty + public String getPathPrefix() { + return pathPrefix; + } + + @JsonProperty + public void setPathPrefix(String pathPrefix) { + this.pathPrefix = pathPrefix; + } + + @JsonProperty + public Credentials getCredentials() { + return credentials; + } + + @JsonProperty + public void setCredentials(Credentials credentials) { + this.credentials = credentials; + } + } + + @JsonProperty + private Multifetch multifetch; + + public static class Multifetch { + + private Integer concurrencyLevel; + private Integer maxCacheSize; + private Boolean enableCacheLogger; + private Integer maxBatchSize; + + @JsonProperty + public Integer getConcurrencyLevel() { + return concurrencyLevel; + } + + @JsonProperty + public void setConcurrencyLevel(Integer concurrencyLevel) { + this.concurrencyLevel = concurrencyLevel; + } + + @JsonProperty + public Integer getMaxCacheSize() { + return maxCacheSize; + } + + @JsonProperty + public void setMaxCacheSize(Integer maxCacheSize) { + this.maxCacheSize = maxCacheSize; + } + + @JsonProperty + public Boolean getEnableCacheLogger() { + return enableCacheLogger; + } + + @JsonProperty + public void setEnableCacheLogger(Boolean enableCacheLogger) { + this.enableCacheLogger = enableCacheLogger; + } + + @JsonProperty + public Integer getMaxBatchSize() { + return maxBatchSize; + } + + @JsonProperty + public void setMaxBatchSize(Integer maxBatchSize) { + this.maxBatchSize = maxBatchSize; + } + } + private boolean enableCors; @NotEmpty @@ -32,15 +185,6 @@ public void setEnableCors(boolean enableCors) { this.enableCors = enableCors; } - @JsonProperty - public double getMaxUsedStoragePercentage() { - return maxUsedStoragePercentage; - } - - @JsonProperty - public void setMaxUsedStoragePercentage(double maxUsedStoragePercentage) { - this.maxUsedStoragePercentage = maxUsedStoragePercentage; - } @JsonProperty public String getCorsOrigins() { @@ -53,12 +197,35 @@ public void setCorsOrigins(String corsOrigins) { } @JsonProperty - public String getFolderPath() { - return folderPath; + public FolderFs getFolderFs() { + return folderFs; + } + + public boolean hasFolderFs() { + return folderFs != null; + } + + @JsonProperty + public void setFolderFs(FolderFs folderFs) { + this.folderFs = folderFs; } @JsonProperty - public void setFolderPath(String folderPath) { - this.folderPath = folderPath; + public S3Fs getS3Fs() { + return s3Fs; + } + + @JsonProperty + public TakipiStorageConfiguration.Multifetch getMultifetch() { + return multifetch; + } + + @JsonProperty + public void setS3Fs(S3Fs s3Fs) { + this.s3Fs = s3Fs; + } + + public boolean hasS3Fs() { + return s3Fs != null; } } diff --git a/src/main/java/com/takipi/oss/storage/TakipiStorageConfigurationEnvResolver.java b/src/main/java/com/takipi/oss/storage/TakipiStorageConfigurationEnvResolver.java new file mode 100644 index 0000000..2c05306 --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/TakipiStorageConfigurationEnvResolver.java @@ -0,0 +1,79 @@ +package com.takipi.oss.storage; + +public class TakipiStorageConfigurationEnvResolver { + private static final int ENV_STATE_NONE = -1; + private static final int ENV_STATE_ESCAPED = 0; + private static final int ENV_STATE_IN_NAME = 1; + + public static String resolveEnv(Object value) { + if (value == null) { + return null; + } + + String property = value.toString(); + + StringBuilder resultBuilder = new StringBuilder(property.length() * 2); + StringBuilder envNameBuilder = new StringBuilder(); + + int envState = ENV_STATE_NONE; + + for (char c : property.toCharArray()) { + switch (envState) { + case ENV_STATE_NONE: { + if (c == '$') + { + envState = ENV_STATE_ESCAPED; + } else { + resultBuilder.append(c); + } + } + break; + + case ENV_STATE_ESCAPED: { + if (c == '{') { + envState = ENV_STATE_IN_NAME; + } else { + resultBuilder.append('$'); + + if (c != '$') { + resultBuilder.append(c); + + envState = ENV_STATE_NONE; + } + } + } + break; + + case ENV_STATE_IN_NAME: { + if (c != '}') { + envNameBuilder.append(c); + } else { + String envVarValue = System.getenv(envNameBuilder.toString()); + + if (envVarValue == null) { + envVarValue = ""; + } + + resultBuilder.append(envVarValue); + + envNameBuilder = new StringBuilder(); + + envState = ENV_STATE_NONE; + } + } + break; + } + } + + if (envState != ENV_STATE_NONE) { + resultBuilder.append('$'); + + if (envState == ENV_STATE_IN_NAME) { + resultBuilder.append('{'); + resultBuilder.append(envNameBuilder); + } + } + + return resultBuilder.toString(); + } +} diff --git a/src/main/java/com/takipi/oss/storage/TakipiStorageMain.java b/src/main/java/com/takipi/oss/storage/TakipiStorageMain.java index c998dae..d9931f7 100644 --- a/src/main/java/com/takipi/oss/storage/TakipiStorageMain.java +++ b/src/main/java/com/takipi/oss/storage/TakipiStorageMain.java @@ -1,17 +1,25 @@ package com.takipi.oss.storage; -import io.dropwizard.Application; -import io.dropwizard.setup.Bootstrap; -import io.dropwizard.setup.Environment; - import java.util.EnumSet; import javax.servlet.DispatcherType; import javax.servlet.FilterRegistration; import org.eclipse.jetty.servlets.CrossOriginFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.takipi.oss.storage.fs.BaseRecord; +import com.takipi.oss.storage.fs.api.Filesystem; +import com.takipi.oss.storage.fs.folder.simple.SimpleFilesystem; +import com.takipi.oss.storage.fs.s3.S3Filesystem; import com.takipi.oss.storage.health.FilesystemHealthCheck; +import com.takipi.oss.storage.resources.diag.MachineInfoOnlyStatusStorageResource; +import com.takipi.oss.storage.resources.diag.NoOpTreeStorageResource; import com.takipi.oss.storage.resources.diag.PingStorageResource; import com.takipi.oss.storage.resources.diag.StatusStorageResource; import com.takipi.oss.storage.resources.diag.TreeStorageResource; @@ -22,7 +30,14 @@ import com.takipi.oss.storage.resources.fs.JsonSimpleFetchStorageResource; import com.takipi.oss.storage.resources.fs.JsonSimpleSearchStorageResource; +import io.dropwizard.Application; +import io.dropwizard.setup.Bootstrap; +import io.dropwizard.setup.Environment; + public class TakipiStorageMain extends Application { + + private final static Logger log = LoggerFactory.getLogger(TakipiStorageMain.class); + public static void main(String[] args) throws Exception { new TakipiStorageMain().run(args); } @@ -38,24 +53,75 @@ public void initialize(Bootstrap bootstrap) { } @Override + @SuppressWarnings({ "rawtypes", "unchecked" }) public void run(TakipiStorageConfiguration configuration, Environment environment) { if (configuration.isEnableCors()) { enableCors(configuration, environment); } - environment.jersey().register(new BinaryStorageResource(configuration)); - environment.jersey().register(new JsonMultiFetchStorageResource(configuration)); - environment.jersey().register(new JsonMultiDeleteStorageResource(configuration)); - - environment.jersey().register(new JsonSimpleFetchStorageResource(configuration)); - environment.jersey().register(new JsonSimpleSearchStorageResource(configuration)); - + Filesystem filesystem = configureFilesystem(configuration, environment); + TakipiStorageConfiguration.Multifetch multifetchConfig = configuration.getMultifetch(); + + environment.healthChecks().register("filesystem", new FilesystemHealthCheck(filesystem)); + environment.jersey().register(new BinaryStorageResource(filesystem)); + environment.jersey().register(new JsonMultiFetchStorageResource(filesystem, multifetchConfig)); + environment.jersey().register(new JsonMultiDeleteStorageResource(filesystem)); + environment.jersey().register(new JsonSimpleFetchStorageResource(filesystem)); + environment.jersey().register(new JsonSimpleSearchStorageResource(filesystem)); environment.jersey().register(new PingStorageResource()); environment.jersey().register(new VersionStorageResource()); + } + + private Filesystem configureFilesystem(TakipiStorageConfiguration configuration, Environment environment) { + if(configuration.hasFolderFs()) { + return configureFolderFilesystem(configuration, environment); + } else if(configuration.hasS3Fs()) { + return configureS3Filesystem(configuration, environment); + } + else { + throw new IllegalArgumentException("Configuration problem. Please configure either folderFs or s3Fs config property."); + } + } + + private SimpleFilesystem configureFolderFilesystem(TakipiStorageConfiguration configuration, Environment environment) { + log.debug("Using local filesystem at: {}", configuration.getFolderFs().getFolderPath()); + environment.jersey().register(new TreeStorageResource(configuration)); environment.jersey().register(new StatusStorageResource(configuration)); - - environment.healthChecks().register("filesystem", new FilesystemHealthCheck(configuration)); + return new SimpleFilesystem(configuration.getFolderFs().getFolderPath(), configuration.getFolderFs().getMaxUsedStoragePercentage()); + } + + private Filesystem configureS3Filesystem(TakipiStorageConfiguration configuration, Environment environment) { + // Setup basically mocked versions of info resources. + environment.jersey().register(new NoOpTreeStorageResource()); + environment.jersey().register(new MachineInfoOnlyStatusStorageResource()); + + // Setup Amazon S3 client + TakipiStorageConfiguration.S3Fs.Credentials credentials = configuration.getS3Fs().getCredentials(); + + AmazonS3 amazonS3; + + if ((credentials.getAccessKey() != null) && + (!credentials.getAccessKey().isEmpty()) && + (credentials.getSecretKey() != null && !credentials.getSecretKey().isEmpty())) { + log.debug("Using S3 Filesystem with keys" ); + + AWSCredentials awsCredentials = new BasicAWSCredentials(credentials.getAccessKey(), credentials.getSecretKey()); + amazonS3 = new AmazonS3Client(awsCredentials); + } + else { + // create a client connection based on IAM role assigned + log.debug("Using S3 Filesystem with IAM Role"); + amazonS3 = new AmazonS3Client(); + } + + // S3 bucket + TakipiStorageConfiguration.S3Fs s3Fs = configuration.getS3Fs(); + String bucket = s3Fs.getBucket(); + String pathPrefix = s3Fs.getPathPrefix(); + log.debug("Using AWS S3 based filesystem with bucket: {}, prefix: {}", bucket, pathPrefix); + + return new S3Filesystem(amazonS3, bucket, pathPrefix); } private void enableCors(TakipiStorageConfiguration configuration, Environment environment) { diff --git a/src/main/java/com/takipi/oss/storage/caching/Cache.java b/src/main/java/com/takipi/oss/storage/caching/Cache.java new file mode 100644 index 0000000..c428377 --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/caching/Cache.java @@ -0,0 +1,79 @@ +package com.takipi.oss.storage.caching; + +public class Cache { + private final CacheDelegator cacheDelegator; + + public Cache(CacheDelegator cacheDelegator) { + this.cacheDelegator = cacheDelegator; + } + + public void put(String key, V value, Serializer serializer) { + if (!canCache()) { + return; + } + + cacheDelegator.put(key, new SerializableCacheValue(value, serializer)); + } + + public void put(String key, byte[] value) { + if (!canCache()) { + return; + } + + cacheDelegator.put(key, new SerializableCacheValue(value, ByteArraySerializer.instance)); + } + + public void put(String key, V value, byte[] serializedValue) { + if (!canCache()) { + return; + } + + cacheDelegator.put(key, new SerializableCacheValue(value, serializedValue)); + } + + public byte[] get(String key) { + if (!canCache()) { + return null; + } + + SerializableCacheValue result = + cacheDelegator.get(key, new SerializableCacheValue(byte[].class, ByteArraySerializer.instance)); + + return result.deserialize(key); + } + + public V get(String key, Class valueClass, Serializer serializer) { + if (!canCache()) { + return null; + } + + SerializableCacheValue result = + cacheDelegator.get(key, new SerializableCacheValue(valueClass, serializer)); + + return result.deserialize(key); + } + + protected boolean canCache() { + return true; + } + + public static interface Serializer { + public byte[] serialize(V value) throws Exception; + + public V deserialize(byte[] bytes) throws Exception; + } + + public static class ByteArraySerializer implements Serializer { + public static ByteArraySerializer instance = new ByteArraySerializer(); + + @Override + public byte[] serialize(byte[] value) throws Exception { + return value; + } + + @Override + public byte[] deserialize(byte[] bytes) throws Exception { + return bytes; + } + } +} diff --git a/src/main/java/com/takipi/oss/storage/caching/CacheDelegator.java b/src/main/java/com/takipi/oss/storage/caching/CacheDelegator.java new file mode 100644 index 0000000..b22d720 --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/caching/CacheDelegator.java @@ -0,0 +1,50 @@ +package com.takipi.oss.storage.caching; + +public abstract class CacheDelegator { + private final CacheDelegator parent; + + public CacheDelegator() { + this(null); + } + + public CacheDelegator(CacheDelegator parent) { + this.parent = parent; + } + + public SerializableCacheValue parentGet(String key, SerializableCacheValue result) { + if (this.parent != null) { + return this.parent.get(key, result); + } + + return result; + } + + public SerializableCacheValue parentPut(String key, SerializableCacheValue value) { + return parentPut(key, value, false); + } + + public SerializableCacheValue parentPut(String key, SerializableCacheValue value, boolean overwrite) { + if (this.parent != null) { + return this.parent.put(key, value, overwrite); + } + + return value; + } + + @Override + public String toString() { + if (this.parent == null) { + return "NUL"; + } + + return this.parent.toString(); + } + + public SerializableCacheValue put(String key, SerializableCacheValue value) { + return put(key, value, false); + } + + public abstract SerializableCacheValue put(String key, SerializableCacheValue value, boolean overwrite); + + public abstract SerializableCacheValue get(String key, SerializableCacheValue result); +} diff --git a/src/main/java/com/takipi/oss/storage/caching/CacheLogger.java b/src/main/java/com/takipi/oss/storage/caching/CacheLogger.java new file mode 100644 index 0000000..cb6728a --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/caching/CacheLogger.java @@ -0,0 +1,88 @@ +package com.takipi.oss.storage.caching; + +import java.util.concurrent.atomic.AtomicLong; + +import com.takipi.oss.storage.helper.StringUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CacheLogger extends CacheDelegator { + private static final Logger logger = LoggerFactory.getLogger(CacheLogger.class); + + private static final AtomicLong counter = new AtomicLong(); + + private final long id; + + public CacheLogger(CacheDelegator parent) { + super(parent); + + this.id = counter.incrementAndGet(); + + logger.info("Cache created (id: {}) {}", this.id, parent); + } + + @Override + public SerializableCacheValue get(String key, SerializableCacheValue result) { + long start = System.currentTimeMillis(); + + try { + return parentGet(key, result); + } + finally { + CacheDelegator delegator = result.getRetriever(); + + if (delegator != null) { + logger.info("{} {}ms {} (retriever: {}) {}", + paddedId(), + paddedDiff(start), + paddedVerb("get"), + paddedDelegator(delegator), + key); + } + } + } + + @Override + public SerializableCacheValue put(String key, SerializableCacheValue value) { + return parentPut(key, value, false); + } + + @Override + public SerializableCacheValue put(String key, SerializableCacheValue value, boolean overwrite) { + long start = System.currentTimeMillis(); + + try { + return parentPut(key, value, overwrite); + } + finally { + CacheDelegator delegator = value.getUpdater(); + + if (delegator != null) { + logger.info("{} {}ms {} (updater: {}) {}", + paddedId(), + paddedDiff(start), + paddedVerb("put"), + paddedDelegator(delegator), + key); + } + } + } + + private String paddedId() { + return StringUtil.padRight(Long.toString(id), 8); + } + + private String paddedVerb(String verb) { + return StringUtil.padRight(verb, 10); + } + + private String paddedDiff(long start) { + long diff = System.currentTimeMillis() - start; + + return StringUtil.padLeft(Long.toString(diff), 8); + } + + private String paddedDelegator(CacheDelegator delegator) { + return StringUtil.padLeft(delegator.getClass().getSimpleName(), 15); + } +} diff --git a/src/main/java/com/takipi/oss/storage/caching/DiskBackedCache.java b/src/main/java/com/takipi/oss/storage/caching/DiskBackedCache.java new file mode 100644 index 0000000..58b0956 --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/caching/DiskBackedCache.java @@ -0,0 +1,262 @@ +package com.takipi.oss.storage.caching; + +import java.io.File; +import java.util.Arrays; +import java.util.Comparator; + +import com.takipi.oss.storage.fs.concurrent.SimpleStopWatch; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DiskBackedCache extends CacheDelegator { + private static final Logger logger = LoggerFactory.getLogger(DiskBackedCache.class); + + private static final HashLock fileAccessLock = new HashLock(); + private static final int MAX_LOGGED_ERRORS = 10; + private static volatile boolean cleanupInProgress = false; + + private final File rootFile; + + private final long totalDiskSpace; + private final long maxDiskSpace; + + private final double cleanupPercentage; + + public DiskBackedCache(File rootFile, double maxDiskUsage, double cleanupPercentage) { + super(null); + + this.rootFile = rootFile; + this.rootFile.mkdirs(); + + this.totalDiskSpace = rootFile.getTotalSpace(); + this.maxDiskSpace = (long) Math.floor(maxDiskUsage * this.totalDiskSpace); + this.cleanupPercentage = cleanupPercentage; + } + + @Override + public String toString() { + return "Disk Cache (path: " + rootFile.getAbsolutePath() + + ") (max: " + maxDiskSpace + + ") (clean: " + cleanupPercentage + ") -> " + super.toString(); + } + + @Override + public SerializableCacheValue get(String key, SerializableCacheValue result) { + try { + return internalGet(key, result); + } + catch (Exception e) { + logger.error("Error getting from disk cache {}", key, e); + return result; + } + } + + public SerializableCacheValue internalGet(String key, SerializableCacheValue result) throws Exception { + File file = keyToFile(key); + + byte[] data = null; + + synchronized (fileAccessLock.get(file)) { + if (!file.canRead()) { + return result; + } + + data = FileUtils.readFileToByteArray(file); + file.setLastModified(System.currentTimeMillis()); + } + + if (data != null) { + result.setSerializedValue(this, data); + } + + return result; + } + + @Override + public SerializableCacheValue put(String key, SerializableCacheValue value, boolean overwrite) { + try { + return internalPut(key, value, overwrite); + } + catch (Exception e) { + logger.error("Error putting to disk cache {}", key, e); + return value; + } + } + + public SerializableCacheValue internalPut(String key, SerializableCacheValue value, boolean overwrite) + throws Exception { + File file = keyToFile(key); + + if (!checkDiskUsage(true)) { + cleanup(); + + if (!checkDiskUsage(true)) { + return value; + } + } + + byte[] data = value.serialize(key); + + if (data == null) { + return value; + } + + synchronized (fileAccessLock.get(file)) { + if ((!overwrite) && + (file.canRead())) { + file.setLastModified(System.currentTimeMillis()); + return value; + } + + checkDiskUsage(false); + + FileUtils.writeByteArrayToFile(file, data); + + if ((!file.setReadable(true, false)) || + (!file.setWritable(true, false))) { + throw new IllegalStateException("Unable to set read/write permissions for local file: " + file.getAbsolutePath()); + } + } + + value.setUpdater(this); + + return value; + } + + public T use(String key, CacheFileCallback callback) { + File file = keyToFile(key); + + if (!file.canRead()) { + return null; + } + + synchronized (fileAccessLock.get(file)) { + if (!file.canRead()) { + return null; + } + + return callback.run(file); + } + } + + private void cleanup() { + if (cleanupPercentage == 0.0) { + logger.info("Disk cleanup is disabled."); + return; + } + + if (cleanupInProgress) { + logger.info("Disk cleanup is already in progress; skipping."); + return; + } + + cleanupInProgress = true; + + logger.info("Starting disk cleanup for {} ({}%).", this, (int) (cleanupPercentage * 100)); + + SimpleStopWatch stopwatch = new SimpleStopWatch(); + + try { + String[] files = rootFile.list(); + + if (files == null) { + return; + } + + logger.info("Sorting {} files by last-modified date.", files.length); + + Arrays.sort(files, new Comparator() { + @Override + public int compare(String filename1, String filename2) { + long date1 = new File(filename1).lastModified(); + long date2 = new File(filename2).lastModified(); + + return Long.valueOf(date1).compareTo(Long.valueOf(date2)); + } + }); + + int deleteCounter = (int) Math.floor(files.length * cleanupPercentage); + + logger.info("About to attempt to delete {} old files.", deleteCounter); + + int errorCounter = 0; + int successCounter = 0; + long sizeCounter = 0l; + + for (int i = 0; i < deleteCounter; i++) { + try { + File file = new File(rootFile, files[i]); + + synchronized (fileAccessLock.get(file)) { + if ((file.canWrite()) && + (!file.isDirectory())) { + long fileSize = file.length(); + + FileUtils.forceDelete(file); + + successCounter++; + sizeCounter += fileSize; + } + } + } + catch (Exception ex) { + errorCounter++; + + if (errorCounter <= MAX_LOGGED_ERRORS) { + logger.error("Deleting {} from disk failed: {}: {}", + files[i], ex.getClass().getSimpleName(), ex.getMessage()); + } + } + } + + if (successCounter == 0) { + logger.warn("No files were deleted from: {}.", rootFile); + } + else { + logger.info("A total of {} files ({} bytes) were deleted from: {}.", + successCounter, sizeCounter, rootFile); + } + + if (errorCounter > 0) { + logger.error("A total of {} files could not be deleted.", errorCounter); + } + } + catch (Exception ex) { + logger.error("Disk cleanup failed.", ex); + } + finally { + logger.info("Disk cleanup completed in {} ms.", stopwatch.elapsed()); + + cleanupInProgress = false; + } + } + + private boolean checkDiskUsage(boolean silent) { + long curFreeSpace = rootFile.getUsableSpace(); + long minFreeSpace = totalDiskSpace - maxDiskSpace; + + if (curFreeSpace <= minFreeSpace) { + if (silent) { + return false; + } + else { + throw new IllegalStateException("Max disk usage limit reached: " + + Double.toString((double) curFreeSpace / totalDiskSpace) + + "% left (limit: " + + Double.toString((double) minFreeSpace / totalDiskSpace) + ")"); + } + } + + return true; + } + + private File keyToFile(String key) { + String validFileName = key.replace(File.separator, ""); + return new File(rootFile, validFileName); + } + + public static interface CacheFileCallback { + T run(File file); + } +} diff --git a/src/main/java/com/takipi/oss/storage/caching/HashLock.java b/src/main/java/com/takipi/oss/storage/caching/HashLock.java new file mode 100644 index 0000000..22daaf8 --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/caching/HashLock.java @@ -0,0 +1,31 @@ +package com.takipi.oss.storage.caching; + +public class HashLock { + private static final int DEFAULT_SIZE = 32; + + private final Object[] locks; + + public HashLock() { + this(DEFAULT_SIZE); + } + + public HashLock(int size) { + locks = new Object[size]; + + for (int i = 0; i < size; i++) { + locks[i] = new Object(); + } + } + + public Object get(Object key) { + int hashCode = key.hashCode(); + + if (hashCode < 0) { + hashCode = (hashCode + 1) + Integer.MAX_VALUE; + } + + int bucket = hashCode % locks.length; + + return locks[bucket]; + } +} diff --git a/src/main/java/com/takipi/oss/storage/caching/InMemoryCache.java b/src/main/java/com/takipi/oss/storage/caching/InMemoryCache.java new file mode 100644 index 0000000..12d4497 --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/caching/InMemoryCache.java @@ -0,0 +1,145 @@ +package com.takipi.oss.storage.caching; + +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; + +import com.google.common.cache.Weigher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; + +public class InMemoryCache extends CacheDelegator { + private static final Logger logger = LoggerFactory.getLogger(InMemoryCache.class); + + private final Cache cache; + private final String description; + + public static class Builder { + private CacheBuilder cacheBuilder = CacheBuilder.newBuilder(); + private CacheDelegator parent = null; + private String description = "Memory Cache"; + + public Builder setParentDelegator(CacheDelegator parent) { + this.parent = parent; + return this; + } + + public Builder setMaxElementCount(int maxElementCount) { + cacheBuilder.maximumSize(maxElementCount); + description += " (max elements = " + maxElementCount + ")"; + return this; + } + + public Builder setMaxSize(long maxSize) { + Weigher weigher = new Weigher() { + @Override + public int weigh(String s, Object o) { + byte[] bytes = (byte[]) o; + return s.length() + bytes.length; + } + }; + cacheBuilder.weigher(weigher); + cacheBuilder.maximumWeight(maxSize); + description += " (max size = " + maxSize + ")"; + return this; + } + + public Builder setExpiry(int expiry, TimeUnit timeUnit) { + cacheBuilder.expireAfterAccess(expiry, timeUnit); + description += " (expiry = " + expiry + " " + timeUnit + ")"; + return this; + } + + public InMemoryCache build() { + Cache cache = cacheBuilder.build(); + return new InMemoryCache(parent, cache, description); + } + } + + private InMemoryCache(CacheDelegator parent, Cache cache, String description) { + super(parent); + + this.cache = cache; + this.description = description; + } + + @Override + public String toString() { + return description + " -> " + super.toString(); + } + + @Override + public SerializableCacheValue get(String key, SerializableCacheValue result) { + try { + return internalGet(key, result); + } + catch (Exception e) { + logger.error("Error getting from in memory cache {}", key, e); + return result; + } + } + + public SerializableCacheValue internalGet(final String key, final SerializableCacheValue result) { + Object value = null; + + try { + value = cache.get(key, new Callable() { + @Override + public Object call() throws Exception { + SerializableCacheValue parentResult = InMemoryCache.this.parentGet(key, result); + + Object resultObject = null; + + if (parentResult != null) { + resultObject = parentResult.deserialize(key); + } + + if (resultObject == null) { + throw new IllegalStateException("Object with key: " + key + "not found in cache"); + } + + return resultObject; + } + }); + } + catch (Exception e) { + } + + if (value != null) { + result.setValue(this, value); + } + + return result; + } + + @Override + public SerializableCacheValue put(String key, SerializableCacheValue value, boolean overwrite) { + try { + // The google cache we are using automatically overwrites values if a key exists + // so we don't need to pass the overwrite param + return internalPut(key, value); + } + catch (Exception e) { + logger.error("Error putting to in memory cache {}", key, e); + + return value; + } + } + + public SerializableCacheValue internalPut(final String key, final SerializableCacheValue value) { + if (cache.getIfPresent(key) != null) { + return value; + } + + value.setUpdater(InMemoryCache.this); + cache.put(key, value.deserialize(key)); + + return InMemoryCache.this.parentPut(key, value); + } + + public long size() { + return cache.size(); + } +} diff --git a/src/main/java/com/takipi/oss/storage/caching/SerializableCacheValue.java b/src/main/java/com/takipi/oss/storage/caching/SerializableCacheValue.java new file mode 100644 index 0000000..daf0987 --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/caching/SerializableCacheValue.java @@ -0,0 +1,110 @@ +package com.takipi.oss.storage.caching; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SerializableCacheValue { + private static final Logger logger = LoggerFactory.getLogger(SerializableCacheValue.class); + + private V value; + private Cache.Serializer serializer; + private byte[] serializedValue; + private Class valueClass; + + private CacheDelegator firstRetriever; + private CacheDelegator lastUpdater; + + public SerializableCacheValue(V value, Cache.Serializer serializer) { + this.value = value; + this.serializer = serializer; + } + + public SerializableCacheValue(Class valueClass, Cache.Serializer serializer) { + this.valueClass = valueClass; + this.serializer = serializer; + } + + public SerializableCacheValue(V value, byte[] serializedValue) { + this.value = value; + this.serializedValue = serializedValue; + } + + public CacheDelegator getRetriever() { + return this.firstRetriever; + } + + public CacheDelegator getUpdater() { + return this.lastUpdater; + } + + public void setUpdater(CacheDelegator updater) { + updateUpdater(updater); + } + + public void setSerializedValue(CacheDelegator retriever, byte[] serializedValue) { + updateRetriever(retriever); + this.serializedValue = serializedValue; + } + + public void setValue(CacheDelegator retriever, Object value) { + if (!valueClass.isInstance(value)) { + logger.error("Error setting value, Type mismatch (expected: {}; found: {}).", + valueClass.getSimpleName(), value.getClass().getSimpleName()); + return; + } + + updateRetriever(retriever); + this.value = valueClass.cast(value); + } + + private void updateRetriever(CacheDelegator retriever) { + if (this.firstRetriever == null) { + this.firstRetriever = retriever; + } + } + + private void updateUpdater(CacheDelegator updater) { + this.lastUpdater = updater; + } + + public V deserialize(String name) { + try { + if (value == null) { + if (serializedValue == null) { + return null; + } + + value = serializer.deserialize(serializedValue); + + if (!valueClass.isInstance(value)) { + logger.error("Error serializing, Type mismatch for '{}' in cache (expected: {}; found: {}).", + name, valueClass.getSimpleName(), value.getClass().getSimpleName()); + + return null; + } + + return valueClass.cast(value); + } + + return value; + } + catch (Exception e) { + logger.error("Error deserializing '{}'", name, e); + return null; + } + } + + public byte[] serialize(String name) { + try { + if (serializedValue == null) { + serializedValue = serializer.serialize(value); + } + + return serializedValue; + } + catch (Exception e) { + logger.error("Error serializing '{}'", name, e); + return null; + } + } +} diff --git a/src/main/java/com/takipi/oss/storage/data/simple/SimpleSearchRequest.java b/src/main/java/com/takipi/oss/storage/data/simple/SimpleSearchRequest.java index 199df63..3bbdb23 100644 --- a/src/main/java/com/takipi/oss/storage/data/simple/SimpleSearchRequest.java +++ b/src/main/java/com/takipi/oss/storage/data/simple/SimpleSearchRequest.java @@ -1,10 +1,31 @@ package com.takipi.oss.storage.data.simple; import com.takipi.oss.storage.data.EncodingType; +import com.takipi.oss.storage.fs.api.SearchRequest; -public class SimpleSearchRequest { +public class SimpleSearchRequest implements SearchRequest { public EncodingType encodingType; public String name; public String baseSearchPath; public boolean preventDuplicates; + + @Override + public EncodingType getEncodingType() { + return encodingType; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getBaseSearchPath() { + return baseSearchPath; + } + + @Override + public boolean shouldPreventDuplicates() { + return preventDuplicates; + } } diff --git a/src/main/java/com/takipi/oss/storage/data/simple/SimpleSearchResponse.java b/src/main/java/com/takipi/oss/storage/data/simple/SimpleSearchResponse.java index cad24b5..1a250f6 100644 --- a/src/main/java/com/takipi/oss/storage/data/simple/SimpleSearchResponse.java +++ b/src/main/java/com/takipi/oss/storage/data/simple/SimpleSearchResponse.java @@ -1,6 +1,8 @@ package com.takipi.oss.storage.data.simple; -public class SimpleSearchResponse { +import com.takipi.oss.storage.fs.api.SearchResult; + +public class SimpleSearchResponse implements SearchResult { String data; String path; @@ -9,10 +11,12 @@ public SimpleSearchResponse(String data, String path) { this.path = path; } + @Override public String getData() { return data; } + @Override public String getPath() { return path; } diff --git a/src/main/java/com/takipi/oss/storage/fs/BaseRecord.java b/src/main/java/com/takipi/oss/storage/fs/BaseRecord.java new file mode 100644 index 0000000..1999772 --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/fs/BaseRecord.java @@ -0,0 +1,11 @@ +package com.takipi.oss.storage.fs; + +public interface BaseRecord { + public String getServiceId(); + + public String getType(); + + public String getKey(); + + public String getPath(); +} diff --git a/src/main/java/com/takipi/oss/storage/fs/Record.java b/src/main/java/com/takipi/oss/storage/fs/Record.java index 4b8dcc7..ad0abc4 100644 --- a/src/main/java/com/takipi/oss/storage/fs/Record.java +++ b/src/main/java/com/takipi/oss/storage/fs/Record.java @@ -1,8 +1,8 @@ package com.takipi.oss.storage.fs; -import org.apache.commons.lang3.StringUtils; +import java.io.File; -public class Record { +public class Record implements BaseRecord { private String serviceId; private String type; private String key; @@ -21,18 +21,28 @@ public static Record newRecord(String serviceId, String type, String key) { return new Record(serviceId, type, key); } + @Override public String getServiceId() { return serviceId; } + @Override public String getType() { return type; } + @Override public String getKey() { return key; } + @Override + public String getPath() { + return (this.getServiceId() + File.separator + + this.getType() + File.separator + + this.getKey()); + } + @Override public String toString() { return "service: " + serviceId + ". type: " + type + ". key: " + key + "."; @@ -51,8 +61,8 @@ public boolean equals(Object obj) { Record objRecord = (Record) obj; - return ((StringUtils.equals(serviceId, objRecord.serviceId)) && - (StringUtils.equals(type, objRecord.type)) && - (StringUtils.equals(key, objRecord.key))); + return ((serviceId.equals(objRecord.serviceId)) && + (type.equals(objRecord.type)) && + (key.equals(objRecord.key))); } } diff --git a/src/main/java/com/takipi/oss/storage/fs/SimplePathRecord.java b/src/main/java/com/takipi/oss/storage/fs/SimplePathRecord.java new file mode 100644 index 0000000..f754478 --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/fs/SimplePathRecord.java @@ -0,0 +1,51 @@ +package com.takipi.oss.storage.fs; + +public class SimplePathRecord implements BaseRecord { + + private final String path; + + private final String[] pathParts; + + public static SimplePathRecord newRecord(String path) { + return new SimplePathRecord(path); + } + + private SimplePathRecord(String path) { + this.path = path.replaceAll("//", "/"); + + this.pathParts = path.split("/", 3); + } + + @Override + public String getPath() { + return path; + } + + @Override + public String getServiceId() { + if (pathParts.length > 0) { + return pathParts[0]; + } + + return ""; + } + + @Override + public String getType() { + if (pathParts.length > 1) { + return pathParts[1]; + } + + return ""; + } + + @Override + public String getKey() { + if (pathParts.length > 2) { + return pathParts[2]; + } + + return ""; + } +} + diff --git a/src/main/java/com/takipi/oss/storage/fs/api/Filesystem.java b/src/main/java/com/takipi/oss/storage/fs/api/Filesystem.java index 2320903..18bdd03 100644 --- a/src/main/java/com/takipi/oss/storage/fs/api/Filesystem.java +++ b/src/main/java/com/takipi/oss/storage/fs/api/Filesystem.java @@ -3,14 +3,16 @@ import java.io.IOException; import java.io.InputStream; -public interface Filesystem extends FilesystemHealth { +import com.takipi.oss.storage.fs.BaseRecord; + +public interface Filesystem extends FilesystemHealth { /** * Put record * * @param record * - the record to save the input stream to - * @param bytes - * - the byte array to save + * @param is + * - the input stream to save * @throws IOException * - if there's an error */ @@ -56,4 +58,33 @@ public interface Filesystem extends FilesystemHealth { * - if there's an error */ long size(T record) throws IOException; + + + /** + * Returns the {@link SearchResult} that matches the search query. + * + * @return + * @throws IOException + * @param request + */ + + /** + * Returns the {@link SearchResult} that matches the search query. + * + * @param searchRequest + * - the search request + * @return + * - the result of the search or null if nothing was found. + * @throws IOException + * - if there's an error + */ + SearchResult search(SearchRequest searchRequest) throws IOException; + + /** + * Convert string path to record object + * + * @param path + * @return record + */ + BaseRecord pathToRecord(String path); } diff --git a/src/main/java/com/takipi/oss/storage/fs/api/SearchRequest.java b/src/main/java/com/takipi/oss/storage/fs/api/SearchRequest.java new file mode 100644 index 0000000..d55df9d --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/fs/api/SearchRequest.java @@ -0,0 +1,15 @@ +package com.takipi.oss.storage.fs.api; + +import com.takipi.oss.storage.data.EncodingType; + +public interface SearchRequest { + + EncodingType getEncodingType(); + + String getName(); + + String getBaseSearchPath(); + + boolean shouldPreventDuplicates(); + +} diff --git a/src/main/java/com/takipi/oss/storage/fs/api/SearchResult.java b/src/main/java/com/takipi/oss/storage/fs/api/SearchResult.java new file mode 100644 index 0000000..9719457 --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/fs/api/SearchResult.java @@ -0,0 +1,9 @@ +package com.takipi.oss.storage.fs.api; + +public interface SearchResult { + + String getData(); + + String getPath(); + +} diff --git a/src/main/java/com/takipi/oss/storage/fs/concurrent/ConcurrentTaskExecutor.java b/src/main/java/com/takipi/oss/storage/fs/concurrent/ConcurrentTaskExecutor.java new file mode 100644 index 0000000..daccf37 --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/fs/concurrent/ConcurrentTaskExecutor.java @@ -0,0 +1,75 @@ +package com.takipi.oss.storage.fs.concurrent; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; + +public class ConcurrentTaskExecutor implements TaskExecutor { + + private static final Logger logger = LoggerFactory.getLogger(ConcurrentTaskExecutor.class); + + private final ExecutorService executorService; + private final AtomicInteger threadCount = new AtomicInteger(); + + public ConcurrentTaskExecutor(int maxThreads) { + + logger.info("ConcurrentTaskExecutor maximum number of threads = {}", maxThreads); + + ThreadFactory threadFactory = new ThreadFactory() { + @Override + public Thread newThread(Runnable r) + { + Thread t = new Thread(r); + t.setDaemon(true); + t.setName("conctaskexec_thread_" + threadCount.incrementAndGet()); + return t; + } + }; + + executorService = Executors.newFixedThreadPool(maxThreads, threadFactory); + } + + @Override + public void execute(List tasks) { + + final int count = tasks.size(); + + if (count == 0) { + return; + } + + logger.debug("Starting concurrent task execute for {} tasks", count); + + SimpleStopWatch stopWatch = new SimpleStopWatch(); + + final List> futures = new ArrayList<>(count - 1); + + for (int i = 1; i < count; ++i) { + Runnable task = tasks.get(i); + futures.add(executorService.submit(task)); + } + + try { + Runnable firstTask = tasks.get(0); + firstTask.run(); + } + catch (Exception e) { + logger.error("Error running task", e); + } + + for (Future future : futures) { + try { + future.get(); + } + catch (Exception e) { + logger.error("Error running task", e); + } + } + + logger.debug("Concurrent task executor executed {} tasks in {} ms", count, stopWatch.elapsed()); + } +} diff --git a/src/main/java/com/takipi/oss/storage/fs/concurrent/SequentialTaskExecutor.java b/src/main/java/com/takipi/oss/storage/fs/concurrent/SequentialTaskExecutor.java new file mode 100644 index 0000000..ecd1eda --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/fs/concurrent/SequentialTaskExecutor.java @@ -0,0 +1,32 @@ +package com.takipi.oss.storage.fs.concurrent; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +public class SequentialTaskExecutor implements TaskExecutor { + + private static final Logger logger = LoggerFactory.getLogger(SequentialTaskExecutor.class); + + @Override + public void execute(List tasks) { + + final int count = tasks.size(); + + SimpleStopWatch stopWatch = new SimpleStopWatch(); + + logger.debug("Starting sequential execute for {} tasks", count); + + for (Runnable task : tasks) { + try { + task.run(); + } + catch (Exception e) { + logger.error("Error running task", e); + } + } + + logger.debug("Sequential task executor executed {} tasks in {} ms", count, stopWatch.elapsed()); + } +} diff --git a/src/main/java/com/takipi/oss/storage/fs/concurrent/SimpleStopWatch.java b/src/main/java/com/takipi/oss/storage/fs/concurrent/SimpleStopWatch.java new file mode 100644 index 0000000..7faf720 --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/fs/concurrent/SimpleStopWatch.java @@ -0,0 +1,18 @@ +package com.takipi.oss.storage.fs.concurrent; + +public class SimpleStopWatch +{ + private long start; + + public SimpleStopWatch() { + reset(); + } + + public long elapsed() { + return System.currentTimeMillis() - start; + } + + public void reset() { + start = System.currentTimeMillis(); + } +} diff --git a/src/main/java/com/takipi/oss/storage/fs/concurrent/TaskExecutor.java b/src/main/java/com/takipi/oss/storage/fs/concurrent/TaskExecutor.java new file mode 100644 index 0000000..ee85d4d --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/fs/concurrent/TaskExecutor.java @@ -0,0 +1,7 @@ +package com.takipi.oss.storage.fs.concurrent; + +import java.util.List; + +public interface TaskExecutor { + void execute(List tasks); +} diff --git a/src/main/java/com/takipi/oss/storage/fs/folder/FolderFilesystem.java b/src/main/java/com/takipi/oss/storage/fs/folder/FolderFilesystem.java index 3d97f38..bb68daa 100644 --- a/src/main/java/com/takipi/oss/storage/fs/folder/FolderFilesystem.java +++ b/src/main/java/com/takipi/oss/storage/fs/folder/FolderFilesystem.java @@ -1,18 +1,12 @@ package com.takipi.oss.storage.fs.folder; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - +import com.takipi.oss.storage.fs.BaseRecord; +import com.takipi.oss.storage.fs.api.Filesystem; import org.apache.commons.io.IOUtils; -import com.takipi.oss.storage.fs.api.Filesystem; +import java.io.*; -public abstract class FolderFilesystem extends FolderFilesystemHealth implements Filesystem { +public abstract class FolderFilesystem extends FolderFilesystemHealth implements Filesystem { public FolderFilesystem(String rootFolder, double maxUsedStoragePercentage) { super(rootFolder, maxUsedStoragePercentage); } diff --git a/src/main/java/com/takipi/oss/storage/fs/folder/record/HashSubfolderFilesystem.java b/src/main/java/com/takipi/oss/storage/fs/folder/record/HashSubfolderFilesystem.java index 7a2c7c5..61cb4df 100644 --- a/src/main/java/com/takipi/oss/storage/fs/folder/record/HashSubfolderFilesystem.java +++ b/src/main/java/com/takipi/oss/storage/fs/folder/record/HashSubfolderFilesystem.java @@ -7,9 +7,9 @@ import com.google.common.base.Charsets; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; -import com.takipi.oss.storage.fs.Record; +import com.takipi.oss.storage.fs.BaseRecord; -public class HashSubfolderFilesystem extends RecordFilesystem { +public abstract class HashSubfolderFilesystem extends RecordFilesystem { private HashFunction func; public HashSubfolderFilesystem(String rootFolder, double maxUsedStoragePercentage) { @@ -19,7 +19,7 @@ public HashSubfolderFilesystem(String rootFolder, double maxUsedStoragePercentag } @Override - protected String buildPath(Record record) { + protected String buildPath(T record) { String key = record.getKey(); String hashKey = hashKey(key); @@ -40,4 +40,6 @@ private String hashKey(String key) { return sb.toString(); } + + } diff --git a/src/main/java/com/takipi/oss/storage/fs/folder/record/RecordFilesystem.java b/src/main/java/com/takipi/oss/storage/fs/folder/record/RecordFilesystem.java index d364eae..cca7f1c 100644 --- a/src/main/java/com/takipi/oss/storage/fs/folder/record/RecordFilesystem.java +++ b/src/main/java/com/takipi/oss/storage/fs/folder/record/RecordFilesystem.java @@ -1,18 +1,27 @@ package com.takipi.oss.storage.fs.folder.record; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -import com.takipi.oss.storage.fs.Record; +import com.google.common.base.Predicate; +import com.takipi.oss.storage.data.simple.SimpleSearchResponse; +import com.takipi.oss.storage.fs.BaseRecord; +import com.takipi.oss.storage.fs.SimplePathRecord; +import com.takipi.oss.storage.fs.api.SearchRequest; +import com.takipi.oss.storage.fs.api.SearchResult; import com.takipi.oss.storage.fs.folder.FolderFilesystem; +import com.takipi.oss.storage.helper.FilesystemUtil; -public class RecordFilesystem extends FolderFilesystem { +public class RecordFilesystem extends FolderFilesystem { public RecordFilesystem(String rootFolder, double maxUsedStoragePercentage) { super(rootFolder, maxUsedStoragePercentage); } @Override - protected String buildPath(Record record) { + protected String buildPath(T record) { Path recordPath = Paths.get(root.getPath(), escape(record.getServiceId()), escape(record.getType()), escape(record.getKey())); @@ -22,4 +31,71 @@ protected String buildPath(Record record) { protected String escape(String value) { return value.replace("..", "__").replace("/", "-").replace("\\", "-"); } + + @Override + public SearchResult search(SearchRequest searchRequest) throws IOException { + File searchRoot = new File(getRoot(), FilesystemUtil.fixPath(searchRequest.getBaseSearchPath())); + + ResourceFileCallback fileCallback = new ResourceFileCallback(searchRequest.getName(), searchRequest.shouldPreventDuplicates()); + FilesystemUtil.listFilesRecursively(searchRoot, fileCallback); + File result = fileCallback.getFoundFile(); + + if (result == null) { + return null; + } + + String relFSPath = result.getAbsolutePath().replace(getRoot().getAbsolutePath(), ""); + String data = FilesystemUtil.encode(new FileInputStream(relFSPath), searchRequest.getEncodingType()); + + if (data == null) { + return null; + } + + return new SimpleSearchResponse(data, relFSPath.replace(searchRequest.getName(), "")); + } + + private static class ResourceFileCallback implements Predicate { + private final String resourceName; + private final boolean preventDuplicates; + + private File foundFile; + + protected ResourceFileCallback(String resourceName, boolean preventDuplicates) + { + this.resourceName = resourceName; + this.preventDuplicates = preventDuplicates; + + this.foundFile = null; + } + + @Override + public boolean apply(File file) + { + if (!resourceName.equals(file.getName())) + { + return false; + } + + if ((preventDuplicates) && + (foundFile != null)) + { + foundFile = null; // never find more than one result if preventing duplicates + return true; + } + + foundFile = file; + + return !preventDuplicates; // if we don't prevent duplicates, we stop right now + } + + public File getFoundFile() + { + return foundFile; + } + } + + @Override + public BaseRecord pathToRecord(String path) { + return SimplePathRecord.newRecord(path); + } } diff --git a/src/main/java/com/takipi/oss/storage/fs/folder/simple/SimpleFilesystem.java b/src/main/java/com/takipi/oss/storage/fs/folder/simple/SimpleFilesystem.java index 058a2cb..d009046 100644 --- a/src/main/java/com/takipi/oss/storage/fs/folder/simple/SimpleFilesystem.java +++ b/src/main/java/com/takipi/oss/storage/fs/folder/simple/SimpleFilesystem.java @@ -1,19 +1,27 @@ package com.takipi.oss.storage.fs.folder.simple; +import java.io.File; +import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; +import com.google.common.base.Predicate; +import com.takipi.oss.storage.data.simple.SimpleSearchResponse; +import com.takipi.oss.storage.fs.BaseRecord; +import com.takipi.oss.storage.fs.SimplePathRecord; +import com.takipi.oss.storage.fs.api.SearchRequest; +import com.takipi.oss.storage.fs.api.SearchResult; import com.takipi.oss.storage.fs.folder.FolderFilesystem; import com.takipi.oss.storage.helper.FilesystemUtil; -public class SimpleFilesystem extends FolderFilesystem { +public class SimpleFilesystem extends FolderFilesystem { public SimpleFilesystem(String rootFolder, double maxUsedStoragePercentage) { super(rootFolder, maxUsedStoragePercentage); } @Override - protected String buildPath(String record) { - Path recordPath = Paths.get(root.getPath(), escape(record)); + protected String buildPath(SimplePathRecord record) { + Path recordPath = Paths.get(root.getPath(), escape(record.getPath())); return recordPath.toString(); } @@ -21,4 +29,71 @@ protected String buildPath(String record) { protected String escape(String value) { return FilesystemUtil.fixPath(value); } + + @Override + public SearchResult search(SearchRequest searchRequest) throws IOException { + File searchRoot = new File(getRoot(), FilesystemUtil.fixPath(searchRequest.getBaseSearchPath())); + + ResourceFileCallback fileCallback = new ResourceFileCallback(searchRequest.getName(), searchRequest.shouldPreventDuplicates()); + FilesystemUtil.listFilesRecursively(searchRoot, fileCallback); + File result = fileCallback.getFoundFile(); + + if (result == null) { + return null; + } + + String relFSPath = result.getAbsolutePath().replace(getRoot().getAbsolutePath(), ""); + String data = FilesystemUtil.read(this, SimplePathRecord.newRecord(relFSPath), searchRequest.getEncodingType()); + + if (data == null) { + return null; + } + + return new SimpleSearchResponse(data, relFSPath.replace(searchRequest.getName(), "")); + } + + private static class ResourceFileCallback implements Predicate { + private final String resourceName; + private final boolean preventDuplicates; + + private File foundFile; + + protected ResourceFileCallback(String resourceName, boolean preventDuplicates) + { + this.resourceName = resourceName; + this.preventDuplicates = preventDuplicates; + + this.foundFile = null; + } + + @Override + public boolean apply(File file) + { + if (!resourceName.equals(file.getName())) + { + return false; + } + + if ((preventDuplicates) && + (foundFile != null)) + { + foundFile = null; // never find more than one result if preventing duplicates + return true; + } + + foundFile = file; + + return !preventDuplicates; // if we don't prevent duplicates, we stop right now + } + + public File getFoundFile() + { + return foundFile; + } + } + + @Override + public BaseRecord pathToRecord(String path) { + return SimplePathRecord.newRecord(path); + } } diff --git a/src/main/java/com/takipi/oss/storage/fs/s3/S3Filesystem.java b/src/main/java/com/takipi/oss/storage/fs/s3/S3Filesystem.java new file mode 100644 index 0000000..86e77ec --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/fs/s3/S3Filesystem.java @@ -0,0 +1,109 @@ +package com.takipi.oss.storage.fs.s3; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.ListObjectsRequest; +import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.takipi.oss.storage.data.simple.SimpleSearchResponse; +import com.takipi.oss.storage.fs.BaseRecord; +import com.takipi.oss.storage.fs.SimplePathRecord; +import com.takipi.oss.storage.fs.api.Filesystem; +import com.takipi.oss.storage.fs.api.SearchRequest; +import com.takipi.oss.storage.fs.api.SearchResult; +import com.takipi.oss.storage.helper.FilesystemUtil; + +public class S3Filesystem implements Filesystem { + + private final AmazonS3 amazonS3; + private final String bucket; + private final String pathPrefix; + + public S3Filesystem(AmazonS3 amazonS3, String bucket, String pathPrefix) { + + this.amazonS3 = amazonS3; + this.bucket = bucket; + this.pathPrefix = pathPrefix; + } + + @Override + public void put(T record, InputStream is) throws IOException { + ObjectMetadata objectMetadata = new ObjectMetadata(); + amazonS3.putObject(bucket, keyOf(record), is, objectMetadata); + } + + @Override + public InputStream get(T record) throws IOException { + return amazonS3.getObject(bucket, keyOf(record)).getObjectContent(); + } + + @Override + public void delete(T record) throws IOException { + amazonS3.deleteObject(bucket, keyOf(record)); + } + + @Override + public boolean exists(T record) throws IOException { + return amazonS3.doesObjectExist(bucket, keyOf(record)); + } + + @Override + public long size(T record) throws IOException { + return amazonS3.getObjectMetadata(bucket, keyOf(record)).getContentLength(); + } + + @Override + public SearchResult search(SearchRequest searchRequest) throws IOException { + + // Start a prefix search + ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); + listObjectsRequest.setBucketName(bucket); + + if (this.pathPrefix != null) { + listObjectsRequest.setPrefix(this.pathPrefix + "/" + searchRequest.getBaseSearchPath()); + } else { + listObjectsRequest.setPrefix(searchRequest.getBaseSearchPath()); + } + + ObjectListing objectListing = amazonS3.listObjects(listObjectsRequest); + + // Just select the first object + S3Object s3Object = null; + for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { + if(objectSummary.getKey().contains(searchRequest.getName())) { + s3Object = amazonS3.getObject(bucket, objectSummary.getKey()); + break; + } + } + + if (s3Object == null) { + return null; + } else { + String data = FilesystemUtil.encode(s3Object.getObjectContent(), searchRequest.getEncodingType()); + return new SimpleSearchResponse(data, searchRequest.getBaseSearchPath()); + } + } + + @Override + public boolean healthy() { + return true; + } + + @Override + public BaseRecord pathToRecord(String path) { + return SimplePathRecord.newRecord(path); + } + + private String keyOf(T record) { + if (this.pathPrefix != null) { + return this.pathPrefix + File.separator + record.getPath(); + } + + return record.getPath(); + } +} diff --git a/src/main/java/com/takipi/oss/storage/health/FilesystemHealthCheck.java b/src/main/java/com/takipi/oss/storage/health/FilesystemHealthCheck.java index 53d4735..fd6c6d9 100644 --- a/src/main/java/com/takipi/oss/storage/health/FilesystemHealthCheck.java +++ b/src/main/java/com/takipi/oss/storage/health/FilesystemHealthCheck.java @@ -1,21 +1,18 @@ package com.takipi.oss.storage.health; import com.codahale.metrics.health.HealthCheck; -import com.takipi.oss.storage.TakipiStorageConfiguration; -import com.takipi.oss.storage.fs.api.FilesystemHealth; -import com.takipi.oss.storage.fs.folder.FolderFilesystemHealth; +import com.takipi.oss.storage.fs.api.Filesystem; public class FilesystemHealthCheck extends HealthCheck { - private final FilesystemHealth fsh; + private final Filesystem filesystem; - public FilesystemHealthCheck(TakipiStorageConfiguration configuration) { - this.fsh = new FolderFilesystemHealth(configuration.getFolderPath(), - configuration.getMaxUsedStoragePercentage()); + public FilesystemHealthCheck(Filesystem filesystem) { + this.filesystem = filesystem; } @Override protected Result check() throws Exception { - if (fsh.healthy()) { + if (filesystem.healthy()) { return Result.healthy(); } else { return Result.unhealthy("Problem with filesystem"); diff --git a/src/main/java/com/takipi/oss/storage/helper/FilesystemUtil.java b/src/main/java/com/takipi/oss/storage/helper/FilesystemUtil.java index 5db3982..cb48cad 100644 --- a/src/main/java/com/takipi/oss/storage/helper/FilesystemUtil.java +++ b/src/main/java/com/takipi/oss/storage/helper/FilesystemUtil.java @@ -14,6 +14,7 @@ import com.google.common.base.Predicate; import com.takipi.oss.storage.data.EncodingType; +import com.takipi.oss.storage.fs.BaseRecord; import com.takipi.oss.storage.fs.api.Filesystem; public class FilesystemUtil { @@ -23,7 +24,7 @@ public static String fixPath(String path) { return path.replace("/", File.separator).replace("\\", File.separator); } - public static String read(Filesystem fs, T record, EncodingType encodingType) { + public static String read(Filesystem fs, T record, EncodingType encodingType) { InputStream is = null; try { @@ -41,7 +42,7 @@ public static String read(Filesystem fs, T record, EncodingType encodingT } } - private static String encode(InputStream is, EncodingType type) throws IOException { + public static String encode(InputStream is, EncodingType type) throws IOException { switch (type) { case PLAIN: case JSON: { diff --git a/src/main/java/com/takipi/oss/storage/helper/StringUtil.java b/src/main/java/com/takipi/oss/storage/helper/StringUtil.java new file mode 100644 index 0000000..666c708 --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/helper/StringUtil.java @@ -0,0 +1,11 @@ +package com.takipi.oss.storage.helper; + +public class StringUtil { + public static String padRight(String str, int targetLength) { + return String.format("%1$-" + targetLength + "s", str); + } + + public static String padLeft(String str, int targetLength) { + return String.format("%1$" + targetLength + "s", str); + } +} diff --git a/src/main/java/com/takipi/oss/storage/resources/diag/MachineInfoOnlyStatusStorageResource.java b/src/main/java/com/takipi/oss/storage/resources/diag/MachineInfoOnlyStatusStorageResource.java new file mode 100644 index 0000000..961938e --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/resources/diag/MachineInfoOnlyStatusStorageResource.java @@ -0,0 +1,49 @@ +package com.takipi.oss.storage.resources.diag; + +import com.codahale.metrics.annotation.Timed; +import com.takipi.oss.storage.data.status.MachineStatus; +import com.takipi.oss.storage.helper.StatusUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.Consumes; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +@Path("/storage/v1/diag/status") +@Consumes(MediaType.TEXT_PLAIN) +@Produces(MediaType.APPLICATION_JSON) +public class MachineInfoOnlyStatusStorageResource { + private static final Logger logger = LoggerFactory.getLogger(MachineInfoOnlyStatusStorageResource.class); + + @POST + @Timed + public Response post() { + try { + MachineStatus machineStatus = new MachineStatus(); + + collectMachineInfo(machineStatus); + + return Response.ok(machineStatus).build(); + } catch (Exception e) { + logger.error("Failed retrieving System Status", e); + return Response.serverError().entity("Failed retrieving System Status").build(); + } + } + + private void collectMachineInfo(MachineStatus machineStatus) { + machineStatus.setMachineName(StatusUtil.getMachineName()); + machineStatus.setPid(StatusUtil.getProcessId()); + machineStatus.setJvmUpTimeMillis(StatusUtil.getJvmUpTimeInMilli()); + machineStatus.setAvailableProcessors(StatusUtil.getAvailableProcessors()); + machineStatus.setLoadAverage(StatusUtil.getLoadAvg()); + machineStatus.setProcessCpuLoad(StatusUtil.getProcessCpuLoad()); + machineStatus.setTotalRamBytes(StatusUtil.getTotalRamInBytes()); + machineStatus.setUsedRamBytes(StatusUtil.getUsedRamInBytes()); + machineStatus.setHeapSizeBytes(StatusUtil.getHeapSizeInBytes()); + machineStatus.setPermGenSizeBytes(StatusUtil.getPermGenSizeInBytes()); + } +} diff --git a/src/main/java/com/takipi/oss/storage/resources/diag/NoOpTreeStorageResource.java b/src/main/java/com/takipi/oss/storage/resources/diag/NoOpTreeStorageResource.java new file mode 100644 index 0000000..9c9f3a6 --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/resources/diag/NoOpTreeStorageResource.java @@ -0,0 +1,23 @@ +package com.takipi.oss.storage.resources.diag; + +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import com.codahale.metrics.annotation.Timed; + +@Path("/storage/v1/diag/tree") +@Consumes(MediaType.TEXT_PLAIN) +@Produces(MediaType.TEXT_PLAIN) +public class NoOpTreeStorageResource { + + @GET + @Timed + public Response get() { + return Response.ok("").build(); + } + +} diff --git a/src/main/java/com/takipi/oss/storage/resources/diag/StatusStorageResource.java b/src/main/java/com/takipi/oss/storage/resources/diag/StatusStorageResource.java index 67c1534..a779b28 100644 --- a/src/main/java/com/takipi/oss/storage/resources/diag/StatusStorageResource.java +++ b/src/main/java/com/takipi/oss/storage/resources/diag/StatusStorageResource.java @@ -44,7 +44,7 @@ public class StatusStorageResource { protected final String folderPath; public StatusStorageResource(TakipiStorageConfiguration configuration) { - this.folderPath = configuration.getFolderPath(); + this.folderPath = configuration.getFolderFs().getFolderPath(); } @POST diff --git a/src/main/java/com/takipi/oss/storage/resources/diag/TreeStorageResource.java b/src/main/java/com/takipi/oss/storage/resources/diag/TreeStorageResource.java index 78c25f6..ee2e80a 100644 --- a/src/main/java/com/takipi/oss/storage/resources/diag/TreeStorageResource.java +++ b/src/main/java/com/takipi/oss/storage/resources/diag/TreeStorageResource.java @@ -29,7 +29,7 @@ public class TreeStorageResource { protected final String folderPath; public TreeStorageResource(TakipiStorageConfiguration configuration) { - this.folderPath = configuration.getFolderPath(); + this.folderPath = configuration.getFolderFs().getFolderPath(); } @GET diff --git a/src/main/java/com/takipi/oss/storage/resources/fs/BinaryStorageResource.java b/src/main/java/com/takipi/oss/storage/resources/fs/BinaryStorageResource.java index 8d13665..8db1cc8 100644 --- a/src/main/java/com/takipi/oss/storage/resources/fs/BinaryStorageResource.java +++ b/src/main/java/com/takipi/oss/storage/resources/fs/BinaryStorageResource.java @@ -22,18 +22,20 @@ import org.slf4j.LoggerFactory; import com.codahale.metrics.annotation.Timed; -import com.takipi.oss.storage.TakipiStorageConfiguration; import com.takipi.oss.storage.fs.Record; -import com.takipi.oss.storage.resources.fs.base.HashFileSystemStorageResource; +import com.takipi.oss.storage.fs.api.Filesystem; @Path("/storage/v1/binary/{serviceId}/{type}/{key:.+}") @Consumes(MediaType.APPLICATION_OCTET_STREAM) @Produces(MediaType.APPLICATION_OCTET_STREAM) -public class BinaryStorageResource extends HashFileSystemStorageResource { +public class BinaryStorageResource { + private static final Logger logger = LoggerFactory.getLogger(BinaryStorageResource.class); - public BinaryStorageResource(TakipiStorageConfiguration configuration) { - super(configuration); + private final Filesystem filesystem; + + public BinaryStorageResource(Filesystem filesystem) { + this.filesystem = filesystem; } @GET @@ -103,7 +105,7 @@ public Response delete(@PathParam("serviceId") @DefaultValue("") String serviceI } try { - fs.delete(Record.newRecord(serviceId, type, key)); + filesystem.delete(Record.newRecord(serviceId, type, key)); return Response.ok().build(); } catch (FileNotFoundException e) { logger.warn("Key not found: {}", key); @@ -115,21 +117,21 @@ public Response delete(@PathParam("serviceId") @DefaultValue("") String serviceI } protected Response internalGet(Record record) throws IOException { - InputStream is = fs.get(record); - - long size = fs.size(record); - + InputStream is = filesystem.get(record); + + long size = filesystem.size(record); + return Response.ok(is).header(HttpHeaders.CONTENT_LENGTH, size).build(); } protected Response internalHead(Record record) throws IOException { - long size = fs.size(record); - + long size = filesystem.size(record); + return Response.ok().header(HttpHeaders.CONTENT_LENGTH, size).build(); } protected Response internalPut(Record record, InputStream is) throws IOException { - fs.put(record, is); + filesystem.put(record, is); return Response.ok().build(); } diff --git a/src/main/java/com/takipi/oss/storage/resources/fs/JsonMultiDeleteStorageResource.java b/src/main/java/com/takipi/oss/storage/resources/fs/JsonMultiDeleteStorageResource.java index 40a9953..e2de986 100644 --- a/src/main/java/com/takipi/oss/storage/resources/fs/JsonMultiDeleteStorageResource.java +++ b/src/main/java/com/takipi/oss/storage/resources/fs/JsonMultiDeleteStorageResource.java @@ -1,6 +1,13 @@ package com.takipi.oss.storage.resources.fs; -import java.util.List; +import com.codahale.metrics.annotation.Timed; +import com.google.common.collect.Lists; +import com.takipi.oss.storage.data.delete.MultiDeleteRequest; +import com.takipi.oss.storage.data.delete.MultiDeleteResponse; +import com.takipi.oss.storage.fs.Record; +import com.takipi.oss.storage.fs.api.Filesystem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.ws.rs.Consumes; import javax.ws.rs.POST; @@ -8,26 +15,19 @@ import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.codahale.metrics.annotation.Timed; -import com.google.common.collect.Lists; -import com.takipi.oss.storage.TakipiStorageConfiguration; -import com.takipi.oss.storage.data.delete.MultiDeleteRequest; -import com.takipi.oss.storage.data.delete.MultiDeleteResponse; -import com.takipi.oss.storage.fs.Record; -import com.takipi.oss.storage.resources.fs.base.HashFileSystemStorageResource; +import java.util.List; @Path("/storage/v1/json/multidelete") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) -public class JsonMultiDeleteStorageResource extends HashFileSystemStorageResource { +public class JsonMultiDeleteStorageResource { + private static final Logger logger = LoggerFactory.getLogger(JsonMultiDeleteStorageResource.class); - public JsonMultiDeleteStorageResource(TakipiStorageConfiguration configuration) { - super(configuration); + private final Filesystem filesystem; + + public JsonMultiDeleteStorageResource(Filesystem filesystem) { + this.filesystem = filesystem; } @POST @@ -47,7 +47,7 @@ private MultiDeleteResponse handleResponse(MultiDeleteRequest request) { for (Record record : request.records) { try { - fs.delete(record); + filesystem.delete(record); deletedRecords.add(record); } catch (Exception e) { logger.error("Problem deleting record " + record, e); diff --git a/src/main/java/com/takipi/oss/storage/resources/fs/JsonMultiFetchStorageResource.java b/src/main/java/com/takipi/oss/storage/resources/fs/JsonMultiFetchStorageResource.java index 5708656..2236bb7 100644 --- a/src/main/java/com/takipi/oss/storage/resources/fs/JsonMultiFetchStorageResource.java +++ b/src/main/java/com/takipi/oss/storage/resources/fs/JsonMultiFetchStorageResource.java @@ -1,6 +1,16 @@ package com.takipi.oss.storage.resources.fs; -import java.util.List; +import com.codahale.metrics.annotation.Timed; +import com.takipi.oss.storage.TakipiStorageConfiguration; +import com.takipi.oss.storage.data.fetch.MultiFetchRequest; +import com.takipi.oss.storage.data.fetch.MultiFetchResponse; +import com.takipi.oss.storage.fs.Record; +import com.takipi.oss.storage.fs.api.Filesystem; +import com.takipi.oss.storage.fs.s3.S3Filesystem; +import com.takipi.oss.storage.resources.fs.multifetcher.MultiFetcher; +import com.takipi.oss.storage.fs.concurrent.TaskExecutor; +import com.takipi.oss.storage.fs.concurrent.ConcurrentTaskExecutor; +import com.takipi.oss.storage.fs.concurrent.SequentialTaskExecutor; import javax.ws.rs.Consumes; import javax.ws.rs.POST; @@ -9,60 +19,48 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.codahale.metrics.annotation.Timed; -import com.google.common.collect.Lists; -import com.takipi.oss.storage.TakipiStorageConfiguration; -import com.takipi.oss.storage.data.RecordWithData; -import com.takipi.oss.storage.data.fetch.MultiFetchRequest; -import com.takipi.oss.storage.data.fetch.MultiFetchResponse; -import com.takipi.oss.storage.fs.Record; -import com.takipi.oss.storage.helper.FilesystemUtil; -import com.takipi.oss.storage.resources.fs.base.HashFileSystemStorageResource; - @Path("/storage/v1/json/multifetch") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) -public class JsonMultiFetchStorageResource extends HashFileSystemStorageResource { - private static final Logger logger = LoggerFactory.getLogger(JsonMultiFetchStorageResource.class); +public class JsonMultiFetchStorageResource { + + private final Filesystem filesystem; + private final MultiFetcher multiFetcher; - public JsonMultiFetchStorageResource(TakipiStorageConfiguration configuration) { - super(configuration); + public JsonMultiFetchStorageResource(Filesystem filesystem, + TakipiStorageConfiguration.Multifetch multifetchConfig) { + + this.filesystem = filesystem; + + TaskExecutor taskExecutor; + + int maxConcurrencyLevel = multifetchConfig.getConcurrencyLevel(); + + if ((filesystem instanceof S3Filesystem) && (maxConcurrencyLevel > 1)) { + taskExecutor = new ConcurrentTaskExecutor(maxConcurrencyLevel); + } + else { + taskExecutor = new SequentialTaskExecutor(); + } + + int maxCacheSize = multifetchConfig.getMaxCacheSize(); + boolean enableCacheLogger = multifetchConfig.getEnableCacheLogger(); + int maxBatchSize = multifetchConfig.getMaxBatchSize(); + + this.multiFetcher = new MultiFetcher(taskExecutor, maxCacheSize, enableCacheLogger, maxBatchSize); } @POST @Timed public Response post(MultiFetchRequest request) { - try { - MultiFetchResponse response = handleResponse(request); + try { + MultiFetchResponse response = multiFetcher.loadData(request, filesystem); return Response.ok(response).build(); - } catch (Exception e) { + } + catch (Exception e) { return Response.serverError().entity("Problem getting keys").build(); } } - private MultiFetchResponse handleResponse(MultiFetchRequest request) { - List records = Lists.newArrayList(); - - for (Record record : request.records) { - try { - String value = FilesystemUtil.read(fs, record, request.encodingType); - - if (value != null) { - records.add(RecordWithData.of(record, value)); - } else { - logger.warn("Key not found: {}", record.getKey()); - } - } catch (Exception e) { - logger.error("Problem with record " + record, e); - } - } - - MultiFetchResponse response = new MultiFetchResponse(records); - - return response; - } } diff --git a/src/main/java/com/takipi/oss/storage/resources/fs/JsonSimpleFetchStorageResource.java b/src/main/java/com/takipi/oss/storage/resources/fs/JsonSimpleFetchStorageResource.java index d02f6b1..8b2947f 100644 --- a/src/main/java/com/takipi/oss/storage/resources/fs/JsonSimpleFetchStorageResource.java +++ b/src/main/java/com/takipi/oss/storage/resources/fs/JsonSimpleFetchStorageResource.java @@ -11,20 +11,23 @@ import org.slf4j.LoggerFactory; import com.codahale.metrics.annotation.Timed; -import com.takipi.oss.storage.TakipiStorageConfiguration; import com.takipi.oss.storage.data.simple.SimpleFetchRequest; import com.takipi.oss.storage.data.simple.SimpleFetchResponse; +import com.takipi.oss.storage.fs.BaseRecord; +import com.takipi.oss.storage.fs.api.Filesystem; import com.takipi.oss.storage.helper.FilesystemUtil; -import com.takipi.oss.storage.resources.fs.base.SimpleFileSystemStorageResource; @Path("/storage/v1/json/simplefetch") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) -public class JsonSimpleFetchStorageResource extends SimpleFileSystemStorageResource { +public class JsonSimpleFetchStorageResource { + private static final Logger logger = LoggerFactory.getLogger(JsonSimpleFetchStorageResource.class); - public JsonSimpleFetchStorageResource(TakipiStorageConfiguration configuration) { - super(configuration); + private final Filesystem filesystem; + + public JsonSimpleFetchStorageResource(Filesystem filesystem) { + this.filesystem = filesystem; } @POST @@ -39,8 +42,8 @@ public Response post(SimpleFetchRequest request) { private Response handleResponse(SimpleFetchRequest request) { try { - String data = FilesystemUtil.read(fs, request.path, request.encodingType); - + String data = FilesystemUtil.read(filesystem, filesystem.pathToRecord(request.path), request.encodingType); + if (data != null) { return Response.ok(new SimpleFetchResponse(data)).build(); } else { diff --git a/src/main/java/com/takipi/oss/storage/resources/fs/JsonSimpleSearchStorageResource.java b/src/main/java/com/takipi/oss/storage/resources/fs/JsonSimpleSearchStorageResource.java index ebd3923..4cf7929 100644 --- a/src/main/java/com/takipi/oss/storage/resources/fs/JsonSimpleSearchStorageResource.java +++ b/src/main/java/com/takipi/oss/storage/resources/fs/JsonSimpleSearchStorageResource.java @@ -1,6 +1,12 @@ package com.takipi.oss.storage.resources.fs; -import java.io.File; +import com.codahale.metrics.annotation.Timed; +import com.takipi.oss.storage.data.simple.SimpleSearchRequest; +import com.takipi.oss.storage.data.simple.SimpleSearchResponse; +import com.takipi.oss.storage.fs.api.Filesystem; +import com.takipi.oss.storage.fs.api.SearchResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.ws.rs.Consumes; import javax.ws.rs.POST; @@ -9,25 +15,17 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.codahale.metrics.annotation.Timed; -import com.google.common.base.Predicate; -import com.takipi.oss.storage.TakipiStorageConfiguration; -import com.takipi.oss.storage.data.simple.SimpleSearchRequest; -import com.takipi.oss.storage.data.simple.SimpleSearchResponse; -import com.takipi.oss.storage.helper.FilesystemUtil; -import com.takipi.oss.storage.resources.fs.base.SimpleFileSystemStorageResource; - @Path("/storage/v1/json/simplesearch") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) -public class JsonSimpleSearchStorageResource extends SimpleFileSystemStorageResource { +public class JsonSimpleSearchStorageResource { + private static final Logger logger = LoggerFactory.getLogger(JsonSimpleSearchStorageResource.class); - public JsonSimpleSearchStorageResource(TakipiStorageConfiguration configuration) { - super(configuration); + private final Filesystem filesystem; + + public JsonSimpleSearchStorageResource(Filesystem filesystem) { + this.filesystem = filesystem; } @POST @@ -42,25 +40,12 @@ public Response post(SimpleSearchRequest request) { private Response handleResponse(SimpleSearchRequest request) { try { - File searchRoot = new File(fs.getRoot(), FilesystemUtil.fixPath(request.baseSearchPath)); - - ResourceFileCallback fileCallback = new ResourceFileCallback(request.name, request.preventDuplicates); - FilesystemUtil.listFilesRecursively(searchRoot, fileCallback); - File result = fileCallback.getFoundFile(); - - if (result == null) { - return searchFailed(request.name); - } - - String relFSPath = result.getAbsolutePath().replace(fs.getRoot().getAbsolutePath(), ""); - String data = FilesystemUtil.read(fs, relFSPath, request.encodingType); - - if (data == null) { + SearchResult searchResult = filesystem.search(request); + if(searchResult != null) { + return Response.ok(new SimpleSearchResponse(searchResult.getData(), searchResult.getPath())).build(); + } else { return searchFailed(request.name); } - - return Response.ok(new SimpleSearchResponse(data, relFSPath.replace(request.name, ""))).build(); - } catch (Exception e) { logger.error("Problem getting: " + request.name, e); return Response.serverError().entity("Problem getting " + request.name).build(); @@ -71,51 +56,5 @@ private Response searchFailed(String name) { logger.warn("File not found: {}", name); return Response.status(404).entity("File not found" + name).build(); } - - private static class ResourceFileCallback implements Predicate - { - private final String resourceName; - private final boolean preventDuplicates; - - private File foundFile; - - protected ResourceFileCallback(String resourceName, boolean preventDuplicates) - { - this.resourceName = resourceName; - this.preventDuplicates = preventDuplicates; - - this.foundFile = null; - } - - @Override - public boolean test(File file) - { - return apply(file); - } - - @Override - public boolean apply(File file) - { - if (!resourceName.equals(file.getName())) - { - return false; - } - - if ((preventDuplicates) && - (foundFile != null)) - { - foundFile = null; // never find more than one result if preventing duplicates - return true; - } - - foundFile = file; - - return !preventDuplicates; // if we don't prevent duplicates, we stop right now - } - - public File getFoundFile() - { - return foundFile; - } - } + } diff --git a/src/main/java/com/takipi/oss/storage/resources/fs/base/FolderFileSystemStorageResource.java b/src/main/java/com/takipi/oss/storage/resources/fs/base/FolderFileSystemStorageResource.java deleted file mode 100644 index e2871a0..0000000 --- a/src/main/java/com/takipi/oss/storage/resources/fs/base/FolderFileSystemStorageResource.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.takipi.oss.storage.resources.fs.base; - -import com.takipi.oss.storage.TakipiStorageConfiguration; -import com.takipi.oss.storage.fs.folder.FolderFilesystem; - -public abstract class FolderFileSystemStorageResource { - protected final FolderFilesystem fs; - - public FolderFileSystemStorageResource(TakipiStorageConfiguration configuration) { - this.fs = getNewFileSystem(configuration); - } - - protected abstract FolderFilesystem getNewFileSystem(TakipiStorageConfiguration configuration); -} diff --git a/src/main/java/com/takipi/oss/storage/resources/fs/base/HashFileSystemStorageResource.java b/src/main/java/com/takipi/oss/storage/resources/fs/base/HashFileSystemStorageResource.java deleted file mode 100644 index 1b3e1fd..0000000 --- a/src/main/java/com/takipi/oss/storage/resources/fs/base/HashFileSystemStorageResource.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.takipi.oss.storage.resources.fs.base; - -import com.takipi.oss.storage.TakipiStorageConfiguration; -import com.takipi.oss.storage.fs.Record; -import com.takipi.oss.storage.fs.folder.FolderFilesystem; -import com.takipi.oss.storage.fs.folder.record.HashSubfolderFilesystem; - -public class HashFileSystemStorageResource extends FolderFileSystemStorageResource { - public HashFileSystemStorageResource(TakipiStorageConfiguration configuration) { - super(configuration); - } - - @Override - protected FolderFilesystem getNewFileSystem(TakipiStorageConfiguration configuration) { - return new HashSubfolderFilesystem(configuration.getFolderPath(), configuration.getMaxUsedStoragePercentage()); - } -} diff --git a/src/main/java/com/takipi/oss/storage/resources/fs/base/SimpleFileSystemStorageResource.java b/src/main/java/com/takipi/oss/storage/resources/fs/base/SimpleFileSystemStorageResource.java deleted file mode 100644 index e529502..0000000 --- a/src/main/java/com/takipi/oss/storage/resources/fs/base/SimpleFileSystemStorageResource.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.takipi.oss.storage.resources.fs.base; - -import com.takipi.oss.storage.TakipiStorageConfiguration; -import com.takipi.oss.storage.fs.folder.FolderFilesystem; -import com.takipi.oss.storage.fs.folder.simple.SimpleFilesystem; - -public class SimpleFileSystemStorageResource extends FolderFileSystemStorageResource { - public SimpleFileSystemStorageResource(TakipiStorageConfiguration configuration) { - super(configuration); - } - - @Override - protected FolderFilesystem getNewFileSystem(TakipiStorageConfiguration configuration) { - return new SimpleFilesystem(configuration.getFolderPath(), configuration.getMaxUsedStoragePercentage()); - } -} diff --git a/src/main/java/com/takipi/oss/storage/resources/fs/multifetcher/MultiFetcher.java b/src/main/java/com/takipi/oss/storage/resources/fs/multifetcher/MultiFetcher.java new file mode 100644 index 0000000..373d3eb --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/resources/fs/multifetcher/MultiFetcher.java @@ -0,0 +1,115 @@ +package com.takipi.oss.storage.resources.fs.multifetcher; + +import com.takipi.oss.storage.data.RecordWithData; +import com.takipi.oss.storage.data.fetch.MultiFetchRequest; +import com.takipi.oss.storage.data.fetch.MultiFetchResponse; +import com.takipi.oss.storage.fs.Record; +import com.takipi.oss.storage.fs.api.Filesystem; +import com.takipi.oss.storage.fs.concurrent.SimpleStopWatch; +import com.takipi.oss.storage.fs.concurrent.TaskExecutor; +import com.takipi.oss.storage.s3cache.S3Cache; +import com.takipi.oss.storage.s3cache.S3CacheImpl; +import com.takipi.oss.storage.s3cache.S3DummyCache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +public class MultiFetcher { + + private static class PartSizeEstimator { + + private static long MIN_LOADED_PARTS_FOR_SIZE_ESTIMATION = 10; + private static int DEFAULT_PART_SIZE_ESTIMATION = 1700; + + private long totalSizeLoaded = 0; + private long numberOfPartsLoaded = 0; + + synchronized void updateStats(long size) { + totalSizeLoaded += size; + ++numberOfPartsLoaded; + } + + synchronized int getEstimatedSizePerPart() { + if (numberOfPartsLoaded < MIN_LOADED_PARTS_FOR_SIZE_ESTIMATION) { + return DEFAULT_PART_SIZE_ESTIMATION; + } + else { + return (int)(totalSizeLoaded / numberOfPartsLoaded); + } + } + } + + private static final Logger logger = LoggerFactory.getLogger(MultiFetcher.class); + + private final TaskExecutor taskExecutor; + private final S3Cache cache; + private final PartSizeEstimator partSizeEstimator = new PartSizeEstimator(); + private final int maxBatchSize; + + public MultiFetcher(TaskExecutor taskExecutor, int maxCacheSize, boolean enableCacheLogger, int maxBatchSize) { + this.taskExecutor = taskExecutor; + this.cache = (maxCacheSize > 0) ? new S3CacheImpl(maxCacheSize, enableCacheLogger) : S3DummyCache.instance; + this.maxBatchSize = maxBatchSize; + } + + public MultiFetchResponse loadData(MultiFetchRequest request, Filesystem filesystem) { + + int estimatedSizePerPart = partSizeEstimator.getEstimatedSizePerPart(); + final int maxBatchCount = maxBatchSize / estimatedSizePerPart; + logger.info("Max batch size = {}. Estimated size per part = {}. Max batch count = {}", + maxBatchSize, estimatedSizePerPart, maxBatchCount); + + List records = request.records; + records = (records.size() > maxBatchCount) ? records.subList(0, maxBatchCount) : records; + + final SimpleStopWatch stopWatch = new SimpleStopWatch(); + final int count = records.size(); + final List recordsWithData = new ArrayList<>(count); + final List recordsToFetch = new ArrayList<>(count); + + logger.debug("Multi fetcher commencing load of {} objects", count); + + long totalSize = 0; + + for (Record record : records) { + String value = cache.get(record.getKey()); + RecordWithData recordWithData = RecordWithData.of(record, value); + recordsWithData.add(recordWithData); + if (value == null) { + recordsToFetch.add(recordWithData); + } + else { + totalSize += value.length(); + logger.debug("Multi fetcher found key {} in cache. {} bytes", record.getKey(), value.length()); + } + } + + if (!recordsToFetch.isEmpty()) { + + final List tasks = new ArrayList<>(recordsToFetch.size()); + + for (RecordWithData recordWithData : recordsToFetch) { + tasks.add(new S3ObjectFetcherTask(recordWithData, filesystem, request.encodingType)); + } + + taskExecutor.execute(tasks); + + for (RecordWithData recordWithData : recordsToFetch) { + String value = recordWithData.getData(); + + if (value != null) { + cache.put(recordWithData.getRecord().getKey(), value); + totalSize += value.length(); + partSizeEstimator.updateStats(value.length()); + } + } + } + + logger.info("Multi fetcher loaded {} parts in {} ms. {} parts found in cache. {} bytes total.", + count, stopWatch.elapsed(), (records.size() - recordsToFetch.size()), totalSize); + + return new MultiFetchResponse(recordsWithData); + } +} diff --git a/src/main/java/com/takipi/oss/storage/resources/fs/multifetcher/S3ObjectFetcherTask.java b/src/main/java/com/takipi/oss/storage/resources/fs/multifetcher/S3ObjectFetcherTask.java new file mode 100644 index 0000000..ba08fab --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/resources/fs/multifetcher/S3ObjectFetcherTask.java @@ -0,0 +1,70 @@ +package com.takipi.oss.storage.resources.fs.multifetcher; + +import com.takipi.oss.storage.data.EncodingType; +import com.takipi.oss.storage.data.RecordWithData; +import com.takipi.oss.storage.fs.Record; +import com.takipi.oss.storage.fs.api.Filesystem; +import com.takipi.oss.storage.fs.concurrent.SimpleStopWatch; +import com.takipi.oss.storage.helper.FilesystemUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class S3ObjectFetcherTask implements Runnable { + + private static final Logger logger = LoggerFactory.getLogger(S3ObjectFetcherTask.class); + + private final RecordWithData recordWithData; + private final Filesystem filesystem; + private final EncodingType encodingType; + + S3ObjectFetcherTask(RecordWithData recordWithData, Filesystem filesystem, EncodingType encodingType) { + this.recordWithData = recordWithData; + this.filesystem = filesystem; + this.encodingType = encodingType; + } + + @Override + public void run() { + String result = load(filesystem, recordWithData.getRecord(), encodingType); + + if (result != null) + { + recordWithData.setData(result); + } + } + + private static String load(Filesystem filesystem, Record record, EncodingType encodingType) { + + final SimpleStopWatch stopWatch = new SimpleStopWatch(); + final String key = record.getKey(); + String value = null; + final int MAX_TRIES = 2; + int count = 0; + + while ((value == null) && (count < MAX_TRIES)) { + + if (count++ > 0) { + logger.warn("Retry loading object for key {}", key); + stopWatch.reset(); + } + + try { + value = FilesystemUtil.read(filesystem, record, encodingType); + } + catch (Exception e) { + // Need this catch because some exceptions inside FilesystemUtil.read are caught and result in a + // null return value, and some are thrown. The code would be simpler if all exceptions were thrown. + } + } + + long elapsed = stopWatch.elapsed(); + + if (value == null) { + logger.error("Failed to load object for key: {}. Elapsed time = {} ms", key, elapsed); + return null; + } + + logger.debug("{} loaded key {} in {} ms. {} bytes", Thread.currentThread().getName(), key, elapsed, value.length()); + return value; + } +} diff --git a/src/main/java/com/takipi/oss/storage/s3cache/S3Cache.java b/src/main/java/com/takipi/oss/storage/s3cache/S3Cache.java new file mode 100644 index 0000000..298a371 --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/s3cache/S3Cache.java @@ -0,0 +1,8 @@ +package com.takipi.oss.storage.s3cache; + +public interface S3Cache { + + String get(String key); + + void put(String key, String value); +} diff --git a/src/main/java/com/takipi/oss/storage/s3cache/S3CacheImpl.java b/src/main/java/com/takipi/oss/storage/s3cache/S3CacheImpl.java new file mode 100644 index 0000000..b1edf39 --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/s3cache/S3CacheImpl.java @@ -0,0 +1,42 @@ +package com.takipi.oss.storage.s3cache; + +import com.takipi.oss.storage.caching.Cache; +import com.takipi.oss.storage.caching.CacheDelegator; +import com.takipi.oss.storage.caching.CacheLogger; +import com.takipi.oss.storage.caching.InMemoryCache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class S3CacheImpl implements S3Cache { + + private static final Logger logger = LoggerFactory.getLogger(S3CacheImpl.class); + + private final Cache cache; + + public S3CacheImpl(long maxSize, boolean enableCacheLogger) { + InMemoryCache memoryCache = new InMemoryCache.Builder().setMaxSize(maxSize).build(); + CacheDelegator cacheDelegator = enableCacheLogger ? new CacheLogger(memoryCache) : memoryCache; + cache = new Cache(cacheDelegator); + } + + public String get(String key) { + + byte[] bytes = cache.get(key); + + if (bytes != null) { + + try { + return new String(bytes, "UTF-8"); + } + catch (Exception e) { + logger.error("Failed to convert byte[] to String", e.getMessage()); + } + } + + return null; + } + + public void put(String key, String value) { + cache.put(key, value.getBytes()); + } +} diff --git a/src/main/java/com/takipi/oss/storage/s3cache/S3DummyCache.java b/src/main/java/com/takipi/oss/storage/s3cache/S3DummyCache.java new file mode 100644 index 0000000..3e7ba03 --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/s3cache/S3DummyCache.java @@ -0,0 +1,20 @@ +package com.takipi.oss.storage.s3cache; + +public class S3DummyCache implements S3Cache { + + public static final S3DummyCache instance = new S3DummyCache(); + + private S3DummyCache() { + + } + + @Override + public String get(String key) { + return null; + } + + @Override + public void put(String key, String value) { + + } +} diff --git a/src/test/java/com/takipi/oss/storage/fs/folder/FolderFilesystemTest.java b/src/test/java/com/takipi/oss/storage/fs/folder/FolderFilesystemTest.java index ea623c1..6757cd8 100644 --- a/src/test/java/com/takipi/oss/storage/fs/folder/FolderFilesystemTest.java +++ b/src/test/java/com/takipi/oss/storage/fs/folder/FolderFilesystemTest.java @@ -25,7 +25,7 @@ public void testRootFolderIsValid() { try { File tempRoot = newTempFolderFile(); - new RecordFilesystem(tempRoot.getPath(), 0.0); + new RecordFilesystem(tempRoot.getPath(), 0.0); } catch (Exception e) { e.printStackTrace(); fail(); @@ -36,7 +36,7 @@ public void testRootFolderIsValid() { @Test public void testRootFolderIsInvalid() { try { - new RecordFilesystem("//:/", 0.0); + new RecordFilesystem("//:/", 0.0); fail(); } catch (Exception e) { } @@ -48,7 +48,7 @@ public void testRootFolderMaxUsedStorageValid() { try { File tempRoot = newTempFolderFile(); - new RecordFilesystem(tempRoot.getPath(), 0.95); + new RecordFilesystem(tempRoot.getPath(), 0.95); } catch (Exception e) { e.printStackTrace(); fail(); @@ -61,7 +61,7 @@ public void testRootFolderMaxUsedStorageBelowZero() { try { File tempRoot = newTempFolderFile(); - new RecordFilesystem(tempRoot.getPath(), -1); + new RecordFilesystem(tempRoot.getPath(), -1); fail(); } catch (Exception e) { } @@ -72,7 +72,7 @@ public void testRootFolderMaxUsedStorageBelowZero() { public void testRootFolderMaxUsedStorageAboveOne() { try { File tempRoot = newTempFolderFile(); - new RecordFilesystem(tempRoot.getPath(), 1.1); + new RecordFilesystem(tempRoot.getPath(), 1.1); fail(); } catch (Exception e) { } @@ -188,7 +188,7 @@ private File newTempFolderFile() { private Filesystem newValidFolderFilesystem() { File temp = newTempFolderFile(); - return new RecordFilesystem(temp.getPath(), 0.99); + return new RecordFilesystem(temp.getPath(), 0.99); } private Record newStubRecord() {