oceanbase
/
oblogproxy
mirror of https://github.com/oceanbase/oblogproxy.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

initial commit

This commit is contained in:
wangxiantong.wxt 2023-11-15 16:32:13 +08:00
commit 597dd81703
144 changed files with 15474 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
.clang-format Normal file
View File

@ -0,0 +1,79 @@
---
Language: Cpp
# BasedOnStyle: LLVM
SortIncludes: false
AccessModifierOffset: -2
AlignAfterOpenBracket: DontAlign
AlignConsecutiveAssignments: false
AlignConsecutiveDeclarations: false
DerivePointerAlignment: false
PointerAlignment: Left
ConstructorInitializerIndentWidth: 4
AlignEscapedNewlinesLeft: true
AlignTrailingComments: true
AllowAllParametersOfDeclarationOnNextLine: true
AllowShortBlocksOnASingleLine: true
AllowShortFunctionsOnASingleLine: false
AllowShortIfStatementsOnASingleLine: false
AllowShortCaseLabelsOnASingleLine: false
AllowShortLoopsOnASingleLine: false
AlwaysBreakTemplateDeclarations: true
BinPackArguments: false
AlwaysBreakBeforeMultilineStrings: false
BreakBeforeBinaryOperators: None
BreakBeforeTernaryOperators: true
BreakConstructorInitializersBeforeComma: false
BinPackParameters: true
ColumnLimit: 120
ConstructorInitializerAllOnOneLineOrOnePerLine: true
DerivePointerBinding: true
ExperimentalAutoDetectBinPacking: false
IndentCaseLabels: true
MaxEmptyLinesToKeep: 1
NamespaceIndentation: None
ObjCSpaceBeforeProtocolList: false
PenaltyBreakBeforeFirstCallParameter: 19
PenaltyBreakComment: 200
PenaltyBreakString: 1000
PenaltyBreakFirstLessLess: 120
PenaltyExcessCharacter: 1000000
PenaltyReturnTypeOnItsOwnLine: 200
AlwaysBreakAfterDefinitionReturnType: None
AlwaysBreakAfterReturnType: None
PointerBindsToType: true
Cpp11BracedListStyle: true
Standard: Auto
IndentWidth: 2
TabWidth: 4
UseTab: Never
BreakBeforeBraces: Custom
BraceWrapping:
AfterClass: false
AfterControlStatement: false
AfterEnum: false
AfterFunction: true
AfterNamespace: false
AfterObjCDeclaration: false
AfterStruct: false
AfterUnion: false
AfterExternBlock: false
BeforeCatch: false
BeforeElse: false
IndentBraces: false
SplitEmptyFunction: false
SplitEmptyRecord: false
SplitEmptyNamespace: false
IndentFunctionDeclarationAfterType: true
SpacesInParentheses: false
SpacesInAngles: false
SpaceInEmptyParentheses: false
SpacesInCStyleCastParentheses: false
SpaceAfterControlStatementKeyword: true
SpaceBeforeAssignmentOperators: true
SpacesBeforeTrailingComments: 2
ContinuationIndentWidth: 4
CommentPragmas: '^lint'
MacroBlockBegin: "
END_CATCH_ERROR$"
...

24
.github/workflows/cmake_build.yml vendored Normal file
View File

@ -0,0 +1,24 @@
name: CMake Build
on:
pull_request:
branches: [ master ]
push:
branches: [ master ]
jobs:
build:
runs-on: ubuntu-latest
container: 'centos:7'
steps:
- uses: actions/checkout@v2
- name: Install dependencies
run: |
curl -fsSLO --compressed https://cmake.org/files/v3.22/cmake-3.22.3-linux-x86_64.tar.gz
tar -zxvf cmake-3.22.3-linux-x86_64.tar.gz -C /usr --strip-components=1 --no-same-owner
yum install -y which git wget rpm rpm-build cpio gcc gcc-c++ make glibc-devel glibc-headers libstdc++-static binutils openssl-devel libaio-devel
yum install -y centos-release-scl
yum install -y devtoolset-9
echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc
- name: Build with cmake
run: scl enable devtoolset-9 'mkdir buildenv && cd buildenv && gcc -v && cmake .. && make -j6'

21
.gitignore vendored Normal file
View File

@ -0,0 +1,21 @@
*.iml
*.swp
.idea
.DS_Store
bin
target
store-log-capture.out
./log
cmake-build-*/
buildenv*/
build/
runenv/
src/common/LogRecord.h
*.pb.*
cmake/

55
.secignore Normal file
View File

@ -0,0 +1,55 @@
##########################################################
# OBFlow Secret Scan Ignore List #
##########################################################
# Above the segmentation lines there are suspected privacy information #
# Please use the file name as the first line and the igored information #
# should be started with space or tab #
# Under the segmentation lines there are the folders which you need to ignore #
##########################################################
**
http://license.coscl.org.cn/MulanPubL-2.0
https://www.oceanbase.com
https://www.oceanbase.com/*
https://open.oceanbase.com
https://open.oceanbase.com/*
https://github.com/oceanbase/oceanbase
https://github.com/oceanbase/oceanbase/issues
https://github.com/oceanbase/*
https://github.com/*
http://mirrors.aliyun.com/oceanbase*
https://mirrors.aliyun.com/oceanbase*
https://github.com/google/*
https://cmake.org/*
http://yum.tbsite.net/*
https://facebook.github.io/*
http://mirrors.aliyun.com/*
https://mirrors.aliyun.com/*
http://mirror.centos.org/*
https://facebook.github.io/*
https://yuque.antfin.com/*
**/*.c
http://www.gnu.org/licenses/
**/*.h
https://dev.mysql.com/*
https://www.openssl.org/*
**/*.cpp
https://dev.mysql.com/*
https://www.openssl.org/*
README.md
https://search.maven.org/*
**/test/*.cpp
http://127.0.0.1:8080/*
--------------------------------------------------------
# Should use GLOB wildcard to configure and analysis the ignored folder
# The root patch should start with '/'
/docs/**
/README.md
/LICENSE
/LEGAL.md
--------------------------------------------------------
# Set the ignored fold to escape the Chinese scan by GLOB wildcard
--------------------------------------------------------
# Set md5 of pemFile string to filter
# This section must be end up with '--------------------------------------------------------'!!!
--------------------------------------------------------

465
CMakeLists.txt Normal file
View File

@ -0,0 +1,465 @@
cmake_minimum_required(VERSION 3.2)
project(oblogproxy CXX)
macro(ob_define VAR DEFAULT)
if (NOT DEFINED ${VAR})
set(${VAR} ${DEFAULT})
endif ()
endmacro()
ob_define(OBLOGPROXY_RELEASEID 1)
option(WITH_DEBUG "With debug symbols" ON)
option(WITH_ASAN "Compile with AddressSanitizer" OFF)
option(WITH_TEST "With Tests" OFF)
option(WITH_DEMO "With Demos" OFF)
option(WITH_JNI_LIB "With oblogreader jni lib" OFF)
option(WITH_GLOG "With google log" ON)
option(WITH_DEPS "With precompiled deps" ON)
option(WITH_LOGMSG "compiled logmsg from source files" OFF)
option(WITH_US_TIMESTAMP "Enable microseconds start timestamp" ON)
option(USE_OBCDC_NS "With libobcdc" ON)
option(USE_LIBOBLOG "With precompiled liboblog" OFF)
option(USE_CXX11_ABI "Build with C++11 ABI" OFF)
option(USE_LIBOBLOG_3 "With liboblog 3.x" OFF)
SET(OMS_PROJECT_SRC_PATH ${CMAKE_CURRENT_SOURCE_DIR})
SET(OMS_PROJECT_BUILD_PATH ${CMAKE_CURRENT_BINARY_DIR})
SET(EXECUTABLE_OUTPUT_PATH ${OMS_PROJECT_BUILD_PATH})
SET(LIBRARY_OUTPUT_PATH ${OMS_PROJECT_BUILD_PATH})
SET(DEP_VAR $ENV{DEP_VAR})
SET(JAVA_HOME $ENV{JAVA_HOME})
message(STATUS "DEP_VAR: ${DEP_VAR}")
message(STATUS "JAVA_HOME: ${JAVA_HOME}")
SET(FIND_LIBOBLOG ON)
if (USE_OBCDC_NS)
SET(OBCDC_NAME "libobcdc")
SET(OBCDC_NAME_VAR "-DUSE_OBCDC_NS")
if (USE_LIBOBLOG_3)
SET(OBCDC_NAME_VAR "${OBCDC_NAME_VAR} -DUSE_LIBOBLOG_3")
endif()
else ()
SET(OBCDC_NAME "liboblog")
endif ()
if (WITH_DEPS)
SET(DEP_VAR ${OMS_PROJECT_BUILD_PATH}/deps)
SET(THIRD_LIB_DIR ${DEP_VAR}/usr/local/oceanbase/deps/devel/lib)
SET(LIBOBLOG_RPM_NAME "oceanbase-ce-cdc")
if (NOT USE_LIBOBLOG)
SET(FIND_LIBOBLOG OFF)
if (USE_LIBOBLOG_3)
# version before 4.0
SET(LIBOBLOG_RPM_NAME "oceanbase-ce-devel")
SET(LIBOBLOG_INCLUDE_PATH ${DEP_VAR}/usr/include)
SET(LIBOBLOG_LIBRARIES ${DEP_VAR}/usr/lib/${OBCDC_NAME}.so)
else()
SET(LIBOBLOG_INCLUDE_PATH "${DEP_VAR}/home/admin/oceanbase/include")
SET(LIBOBLOG_LIBRARIES ${DEP_VAR}/home/admin/oceanbase/lib64/${OBCDC_NAME}.so)
SET(LIBOBLOG_LIB_DIR ${DEP_VAR}/home/admin/oceanbase/lib64/)
SET(LIBOBLOG_DEPS
${THIRD_LIB_DIR}/libaio.so
${THIRD_LIB_DIR}/mariadb/libmariadb.so
)
endif()
endif ()
execute_process(
COMMAND bash deps/dep_create.sh ${OMS_PROJECT_BUILD_PATH}/deps ${LIBOBLOG_RPM_NAME}
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
COMMAND_ERROR_IS_FATAL ANY
)
# make openssl found
CMAKE_POLICY(SET CMP0074 NEW)
SET(OpenSSL_ROOT ${DEP_VAR}/usr/local/oceanbase/deps/devel/)
# compiler
SET(COMPILER_DIR ${DEP_VAR}/usr/local/oceanbase/devtools/bin/)
else ()
# compiler
execute_process(
COMMAND which gcc
OUTPUT_VARIABLE GCC_BIN
)
GET_FILENAME_COMPONENT(COMPILER_DIR ${GCC_BIN} DIRECTORY)
endif ()
message(STATUS "COMPILER_DIR: ${COMPILER_DIR}")
add_compile_options($<$<COMPILE_LANGUAGE:CXX,C>:-pipe>)
add_compile_options($<$<COMPILE_LANGUAGE:CXX,C>:-fPIC>)
add_compile_options($<$<COMPILE_LANGUAGE:CXX,C>:-pie>)
add_compile_options($<$<COMPILE_LANGUAGE:CXX,C>:-znoexecstack>)
add_compile_options($<$<COMPILE_LANGUAGE:CXX,C>:-znow>)
add_compile_options($<$<COMPILE_LANGUAGE:CXX,C>:-fstack-protector-strong>)
add_link_options($<$<COMPILE_LANGUAGE:CXX,C>:-Wl,-z,relro,-z,now>)
find_program(CC NAMES gcc PATHS ${COMPILER_DIR} /usr/bin/ NO_DEFAULT_PATH)
find_program(CXX NAMES g++ PATHS ${COMPILER_DIR} /usr/bin/ NO_DEFAULT_PATH)
find_program(AR NAMES gcc-ar ar PATHS ${COMPILER_DIR} /usr/bin/ NO_DEFAULT_PATH)
SET(CMAKE_C_COMPILER ${CC})
SET(CMAKE_CXX_COMPILER ${CXX})
SET(CMAKE_C_COMPILER_AR ${AR})
SET(CMAKE_CXX_COMPILER_AR ${AR})
message(STATUS "C compiler: ${CMAKE_C_COMPILER}")
message(STATUS "C++ compiler: ${CMAKE_CXX_COMPILER}")
message(STATUS "AR compiler: ${CMAKE_C_COMPILER_AR}")
GET_FILENAME_COMPONENT(COMPILER_DIR ${CMAKE_C_COMPILER} DIRECTORY)
GET_FILENAME_COMPONENT(COMPILER_BASE_DIR ${COMPILER_DIR} DIRECTORY)
SET(CXX_LIB_DIR ${COMPILER_BASE_DIR}/lib64/)
# make protoc found libstdc++
SET(PROTOC_LINK_FLAGS "-L${CXX_LIB_DIR} -static-libstdc++")
if (THIRD_LIB_DIR)
SET(PROTOC_LINK_FLAGS "${PROTOC_LINK_FLAGS} -L${THIRD_LIB_DIR}")
endif ()
SET(DEP_LIB_PATH ${THIRD_LIB_DIR} ${CXX_LIB_DIR})
if (NOT USE_CXX11_ABI)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
endif ()
SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
SET(THIRD_PARTY_PATH ${CMAKE_CURRENT_BINARY_DIR}/third-party)
SET(THIRD_PARTY_BUILD_TYPE Release)
SET(EXTERNAL_PROJECT_LOG_ARGS
LOG_DOWNLOAD 0
LOG_UPDATE 0
LOG_CONFIGURE 0
LOG_BUILD 0
LOG_TEST 0
LOG_INSTALL 0)
INCLUDE(ProcessorCount)
ProcessorCount(NUM_OF_PROCESSOR)
message(NUM_OF_PROCESSOR: ${NUM_OF_PROCESSOR})
#thread
include(FindThreads)
#openssl
find_package(OpenSSL REQUIRED)
message(STATUS "ssl:" ${OPENSSL_SSL_LIBRARY})
message(STATUS "crypto:" ${OPENSSL_CRYPTO_LIBRARY})
ADD_LIBRARY(ssl SHARED IMPORTED GLOBAL)
SET_PROPERTY(TARGET ssl PROPERTY IMPORTED_LOCATION ${OPENSSL_SSL_LIBRARY})
ADD_LIBRARY(crypto SHARED IMPORTED GLOBAL)
SET_PROPERTY(TARGET crypto PROPERTY IMPORTED_LOCATION ${OPENSSL_CRYPTO_LIBRARY})
if (WITH_JNI_LIB)
SET(JAVA_INCLUDE_DIR ${JAVA_HOME}/include ${JAVA_HOME}/include/linux ${JAVA_HOME}/include/darwin)
SET(JAVA_LIB_DIR ${JAVA_HOME}/jre/lib)
else ()
SET(JAVA_INCLUDE_DIR "")
SET(JAVA_LIB_DIR "")
endif ()
include(lz4)
include(jsoncpp)
include(libevent)
if (WITH_GLOG)
include(gflags)
include(glog)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DWITH_GLOG=1")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DWITH_GLOG=1")
else ()
SET(GFLAGS_INCLUDE_DIR "")
SET(GLOG_INCLUDE_DIR "")
SET(GFLAGS_LIBRARIES "")
SET(GLOG_LIBRARIES "")
endif ()
if (WITH_TEST)
include(gtest)
else ()
SET(GTEST_INCLUDE_DIR "")
SET(GTEST_LIBRARIES "")
endif ()
#protobuf
include(protobuf)
file(GLOB PROTO_FILES ${CMAKE_SOURCE_DIR}/proto/*.proto)
message("protoc: ${PROTOBUF_PROTOC_EXECUTABLE}, proto inc: ${PROTOBUF_INCLUDE_DIRS}, lib: ${PROTOBUF_LIBRARIES}, ${PROTOBUF_PROTOC_LIBRARY}, protos: ${PROTO_FILES}")
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/proto)
foreach (PROTO ${PROTO_FILES})
message(proto : ${PROTO})
get_filename_component(PROTO_WE ${PROTO} NAME_WE)
list(APPEND PROTO_HDRS "${CMAKE_CURRENT_BINARY_DIR}/proto/${PROTO_WE}.pb.h")
list(APPEND PROTO_SRCS "${CMAKE_CURRENT_BINARY_DIR}/proto/${PROTO_WE}.pb.cc")
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/proto/${PROTO_WE}.pb.h ${CMAKE_CURRENT_BINARY_DIR}/proto/${PROTO_WE}.pb.cc
COMMAND ${PROTOBUF_PROTOC_EXECUTABLE}
--cpp_out=${CMAKE_CURRENT_BINARY_DIR}/proto
--proto_path=${PROTOBUF_INCLUDE_DIR}
--proto_path=${CMAKE_SOURCE_DIR}/proto ${PROTO}
DEPENDS protobuf
)
endforeach ()
add_library(PROTO_OBJS OBJECT ${PROTO_SRCS} ${PROTO_HDRS})
message("protoc: ${PROTOBUF_PROTOC_EXECUTABLE}, proto srcs : ${PROTO_SRCS}")
# oblog
if (FIND_LIBOBLOG)
find_path(LIBOBLOG_INCLUDE_PATH NAMES ${OBCDC_NAME}.h ${OBCDC_NAME}/${OBCDC_NAME}.h)
find_library(LIBOBLOG_LIBRARIES NAMES ${OBCDC_NAME}.so)
if ((NOT LIBOBLOG_INCLUDE_PATH) OR (NOT LIBOBLOG_LIBRARIES))
message(FATAL_ERROR "Fail to find ${OBCDC_NAME}")
endif ()
endif ()
option(LOGMSG_BY_LIBOBLOG "use logmsg provide by liboblog" ON)
set(LOGMSG_BY_LIBOBLOG_DEFINE "")
if (LOGMSG_BY_LIBOBLOG)
set(LOGMSG_BY_LIBOBLOG_DEFINE "-DLOGMSG_BY_LIBOBLOG=1 -DOB_BUILD_OPENSOURCE=1")
SET(OBLOGMSG_MAPPING "${LOGMSG_BY_LIBOBLOG_DEFINE} -DLogMsgLocalInit=\"if((_t_s_lmb=new(std::nothrow)LogMsgBuf())==nullptr){OMS_ERROR<<\\\"Failed to alloc LogMsgBuf\\\";stop();return;}\" -DLogMsgLocalDestroy=\"delete _t_s_lmb\"")
SET(OBLOGMSG_INCLUDE_DIR ${LIBOBLOG_INCLUDE_PATH} ${LIBOBLOG_INCLUDE_PATH}/oblogmsg ${LIBOBLOG_INCLUDE_PATH}/../oblogmsg)
SET(OBLOGMSG_LIBRARIES ${LIBOBLOG_LIBRARIES})
GET_FILENAME_COMPONENT(OBLOGMSG_LIB_DIR ${OBLOGMSG_LIBRARIES} DIRECTORY)
ADD_LIBRARY(oblogmsg STATIC IMPORTED GLOBAL)
SET_PROPERTY(TARGET oblogmsg PROPERTY IMPORTED_LOCATION ${OBLOGMSG_LIBRARIES})
else ()
include(oblogmsg)
endif ()
GET_FILENAME_COMPONENT(LIBOBLOG_INCLUDE_PATH_NAME ${LIBOBLOG_INCLUDE_PATH} NAME)
if (${LIBOBLOG_INCLUDE_PATH_NAME} STREQUAL "${OBCDC_NAME}")
set(LIBOBLOG_INCLUDE_PATH ${LIBOBLOG_INCLUDE_PATH} ${LIBOBLOG_INCLUDE_PATH}/..)
else ()
set(LIBOBLOG_INCLUDE_PATH ${LIBOBLOG_INCLUDE_PATH} ${LIBOBLOG_INCLUDE_PATH}/${OBCDC_NAME})
endif ()
if (WITH_US_TIMESTAMP)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DWITH_US_TIMESTAMP")
endif()
message("oblogmsg: ${OBLOGMSG_INCLUDE_DIR}, ${OBLOGMSG_LIBRARIES}")
message("liboblog: ${LIBOBLOG_INCLUDE_PATH}, ${LIBOBLOG_LIBRARIES}, ${LIBOBLOG_DEPS}")
execute_process(
COMMAND git log -1 --format=%H
OUTPUT_VARIABLE GIT_VERSION
OUTPUT_STRIP_TRAILING_WHITESPACE
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
)
if (NOT GIT_VERSION)
message(WARNING "oblogproxy fetch git version empty, use current time as program version")
STRING(TIMESTAMP GIT_VERSION "%Y-%m-%d_%H:%M:%S")
endif ()
if (NOT GIT_VERSION)
message(WARNING "oblogproxy fetch current time failed")
SET(GIT_VERSION "2.0.0")
endif ()
message("oblogproxy version: ${GIT_VERSION}")
if (WITH_DEBUG)
SET(DEBUG_SYMBOL "-ggdb")
else ()
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNDEBUG")
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DNDEBUG")
endif ()
if (WITH_ASAN)
SET(ASAN_COMPILE_OPTION "-fsanitize=address -fno-omit-frame-pointer")
SET(ASAN_LINK_OPTION "-fsanitize=address")
endif ()
######### get arch ###########################################
execute_process(COMMAND uname -m
OUTPUT_VARIABLE ARCH
OUTPUT_STRIP_TRAILING_WHITESPACE
COMMAND_ERROR_IS_FATAL ANY
)
set(OS_ARCH ${ARCH})
message(STATUS "os arch ${OS_ARCH}") # x86_64 or aarch64
SET(CMAKE_CXX_STANDARD 17)
SET(CMAKE_CXX_STANDARD_REQUIRED ON)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -static-libstdc++ ${DEBUG_SYMBOL} -pipe -Wall -fPIC -Wno-reorder ${ASAN_COMPILE_OPTION} -D__OMS_VERSION__=\\\"${GIT_VERSION}\\\" ${OBLOGMSG_MAPPING} ${OBCDC_NAME_VAR}")
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 ${DEBUG_SYMBOL} -pipe -Wall -fPIC ${ASAN_COMPILE_OPTION} -D__STDC_LIMIT_MACROS -D__OMS_VERSION__=\\\"${GIT_VERSION}\\\" ${OBLOGMSG_MAPPING} ${OBCDC_NAME_VAR}")
if(OS_ARCH STREQUAL "x86_64")
add_compile_options($<$<COMPILE_LANGUAGE:CXX,C>:-m64>)
endif()
if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
SET(PLATFORM_SPEC rt)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc")
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -static-libgcc")
elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin")
SET(PLATFORM_SPEC ${DEP_LIB}
"-framework CoreFoundation"
"-framework CoreGraphics"
"-framework CoreData"
"-framework CoreText"
"-framework Security"
"-framework Foundation"
"-Wl,-U,_MallocExtension_ReleaseFreeMemory"
"-Wl,-U,_ProfilerStart"
"-Wl,-U,_ProfilerStop")
endif ()
########### start define deps #############################################################################################
set(DEP_INC
${JAVA_INCLUDE_DIR}
${OBLOGMSG_INCLUDE_DIR}
${LZ4_INCLUDE_DIR}
${JSONCPP_INCLUDE_DIR}
${LIBEVENT_INCLUDE_DIR}
${GFLAGS_INCLUDE_DIR}
${GLOG_INCLUDE_DIR}
${PROTOBUF_INCLUDE_DIR}
${GTEST_INCLUDE_DIR}
${OPENSSL_INCLUDE_DIR}
)
set(DEP_LIB_PATH
${DEP_LIB_PATH}
${JAVA_LIB_DIR}
${OBLOGMSG_LIB_DIR}
${LIBOBLOG_LIB_DIR}
${CMAKE_CURRENT_BINARY_DIR}
)
set(DEP_LIBS
${OBLOGMSG_LIBRARIES}
${GTEST_LIBRARIES}
${GLOG_LIBRARIES}
${GFLAGS_LIBRARIES}
${PROTOBUF_LIBRARIES}
${PROTOBUF_PROTOC_LIBRARY}
${LIBEVENT_LIBRARIES}
${JSONCPP_LIBRARIES}
${LZ4_LIBRARIES}
${OPENSSL_LIBRARIES}
m
${PLATFORM_SPEC}
${CMAKE_THREAD_LIBS_INIT}
)
message("DEP_INC: ${DEP_INC}")
message("DEP_LIB_PATH: ${DEP_LIB_PATH}")
message("DEP_LIBS: ${DEP_LIBS}")
########### end define deps ###############################################################################################
# common
set(COMMON_INC
${DEP_INC}
${CMAKE_CURRENT_BINARY_DIR}/proto
./src
./src/common/
)
file(GLOB COMMON_SRC
./src/common/*.cpp
./src/common/*.hpp
./src/codec/*.cpp
./src/communication/*.cpp
./src/obaccess/*.cpp
./src/metric/*.cpp
)
add_library(common STATIC ${COMMON_SRC} $<TARGET_OBJECTS:PROTO_OBJS>)
add_dependencies(common oblogmsg lz4 jsoncpp libevent protobuf)
if (WITH_GLOG)
add_dependencies(common glog)
endif ()
target_include_directories(common PUBLIC ${COMMON_INC})
if (USE_OBCDC_NS)
set(DEP_OBCDC_LIB obcdc ${LIBOBLOG_DEPS})
else ()
set(DEP_OBCDC_LIB oblog ${LIBOBLOG_DEPS})
endif ()
# oblogreader
set(OBLOGREADER_INC ${COMMON_INC} ./src/oblogreader/ ${LIBOBLOG_INCLUDE_PATH})
file(GLOB OBLOGREADER_SRC ./src/oblogreader/*.cpp)
add_library(oblogreader_static STATIC ${OBLOGREADER_SRC})
add_dependencies(oblogreader_static common oblogmsg)
set_target_properties(oblogreader_static PROPERTIES OUTPUT_NAME "oblogreader")
target_include_directories(oblogreader_static PUBLIC ${OBLOGREADER_INC})
target_link_directories(oblogreader_static PUBLIC ${DEP_LIB_PATH})
target_link_libraries(oblogreader_static libcommon.a ${DEP_OBCDC_LIB} ${DEP_LIBS})
# oblogreader_jni
if (WITH_JNI_LIB)
set(OBLOGREADER_JNI_INC ${OBLOGREADER_INC} src/jni)
file(GLOB OBLOGREADER_JNI_SRC src/jni/*.cpp)
add_library(oblogreader_jni STATIC ${OBLOGREADER_JNI_SRC})
add_dependencies(oblogreader_jni oblogreader_static)
target_include_directories(oblogreader_jni PUBLIC ${OBLOGREADER_JNI_INC})
target_link_directories(oblogreader_jni PUBLIC ${DEP_LIB_PATH})
target_link_libraries(oblogreader_jni liboblogreader.a libcommon.a ${DEP_OBCDC_LIB} ${DEP_LIBS})
endif ()
# logproxy static
set(LOGPROXY_INC ${OBLOGREADER_INC} src/arranger/)
file(GLOB LOGPROXY_SRC ./src/arranger/*.cpp)
message("SRC: ${LOGPROXY_SRC}")
add_library(logproxy_static STATIC ${LOGPROXY_SRC})
add_dependencies(logproxy_static oblogreader_static oblogmsg)
set_target_properties(logproxy_static PROPERTIES OUTPUT_NAME "logproxy")
target_include_directories(logproxy_static PUBLIC ${LOGPROXY_INC})
target_link_directories(logproxy_static PUBLIC ${DEP_LIB_PATH})
target_link_libraries(logproxy_static liboblogreader.a libcommon.a ${DEP_OBCDC_LIB} ${DEP_LIBS})
SET(BASE_LIBS liblogproxy.a liboblogreader.a libcommon.a)
# logproxy
add_executable(logproxy ./src/entry.cpp)
add_dependencies(logproxy logproxy_static)
target_include_directories(logproxy PUBLIC ${DEP_INC} ${LOGPROXY_INC})
target_link_directories(logproxy PUBLIC ${DEP_LIB_PATH})
target_link_libraries(logproxy ${BASE_LIBS} ${DEP_OBCDC_LIB} ${DEP_LIBS})
target_link_options(logproxy PUBLIC -static-libstdc++ ${ASAN_LINK_OPTION})
target_link_options(logproxy PRIVATE -pie)
if (WITH_DEMO)
# demo client
file(GLOB DEMO_CLIENT_SRC ./src/demo/client_demo.cpp)
add_executable(demo_client ${DEMO_CLIENT_SRC})
add_dependencies(demo_client common)
target_include_directories(demo_client PUBLIC ${COMMON_INC})
target_link_directories(demo_client PUBLIC ${DEP_LIB_PATH})
target_link_libraries(demo_client libcommon.a ${DEP_LIBS})
target_link_options(demo_client PUBLIC -static-libstdc++ ${ASAN_LINK_OPTION})
endif ()
if (WITH_TEST)
# test_base
file(GLOB TEST_BASE_SRC ./src/test/test_entry.cpp)
add_executable(test_base ${TEST_BASE_SRC})
add_dependencies(test_base common oblogmsg gtest)
target_include_directories(test_base PUBLIC ${LOGPROXY_INC})
target_link_directories(test_base PUBLIC ${DEP_LIB_PATH})
target_link_libraries(test_base libcommon.a ${DEP_LIBS})
target_link_options(test_base PUBLIC -static-libstdc++ ${ASAN_LINK_OPTION})
# test_oblogreader
# file(GLOB TEST_OBLOGREADER_SRC ./src/test/test_oblogreader.cpp)
# add_executable(test_oblogreader ${TEST_OBLOGREADER_SRC})
# add_dependencies(test_oblogreader logproxy_static gtest)
# target_include_directories(test_oblogreader PUBLIC ${LOGPROXY_INC})
# target_link_directories(test_oblogreader PUBLIC ${DEP_LIB_PATH})
# target_link_libraries(test_oblogreader ${BASE_LIBS} ${DEP_OBCDC_LIB} ${DEP_LIBS})
# target_link_options(test_oblogreader PUBLIC ${ASAN_LINK_OPTION})
endif ()
include(rpm)

180
LICENSE Normal file
View File

@ -0,0 +1,180 @@
木兰公共许可证, 第2版
木兰公共许可证， 第2版
2021年5月 http://license.coscl.org.cn/MulanPubL-2.0
您对“贡献”的复制、使用、修改及分发受木兰公共许可证,第2版（以下简称“本许可证”）的如下条款的约束：
0. 定义
“贡献” 是指由“贡献者”许可在“本许可证”下的受版权法保护的作品，包括最初“贡献者”许可在“本许可证”下的作品及后续“贡献者”许可在“本许可证”下的“衍生作品”。
“贡献者” 是指将受版权法保护的作品许可在“本许可证”下的自然人或“法人实体”。
“法人实体” 是指提交贡献的机构及其“关联实体”。
“关联实体” 是指，对“本许可证”下的行为方而言，控制、受控制或与其共同受控制的机构，此处的“控制”是指拥有受控方或共同受控方至少50%直接或间接的投票权、资金或其他有价证券。
“衍生作品” 是指基于“贡献”创作的作品，具体包括对全部或部分“贡献”进行修改、重写、翻译、注释、组合或与之链接（包括动态链接或静态链接）而形成的作品。仅与“贡献”进行进程间通信或系统调用的作品是独立作品，不属于“衍生作品”。
“对应源代码” 是指生成、安装和(对于可执行作品)运行目标代码所需的所有源文件和与之关联的接口定义文件，以及控制这些活动的脚本，但不包括编译环境、编译工具、云服务平台（如果有）。
“分发” 是指通过任何媒介向他人提供“贡献”或“衍生作品”的行为，以及利用“贡献”或“衍生作品”通过网络远程给用户提供服务的行为，例如：通过利用“贡献”或“衍生作品”搭建的云服务平台提供在线服务的行为。
1. 授予版权许可
每个“贡献者”根据“本许可证”授予您永久性的、全球性的、免费的、非独占的、不可撤销的版权许可，您可以复制、使用、修改、“分发”其“贡献”或“衍生作品”，不论修改与否。
2. 授予专利许可
每个“贡献者”根据“本许可证”授予您永久性的、全球性的、免费的、非独占的、不可撤销的（根据本条规定撤销的情形除外）专利许可，供您使用、制造、委托制造、销售、许诺销售、进口其“贡献”或以其他方式转移其“贡献”。前述专利许可仅限于“贡献者”现在或将来拥有或控制的其“贡献”中的专利权利要求，而不包括仅因您对“贡献”的修改而将必然会侵犯到的专利权利要求。如果您或您的“关联实体”直接或间接地，就“贡献”对任何人发起专利侵权诉讼（包括在诉讼中提出反诉请求或交叉请求）或发起其他专利维权行动，则“贡献者”根据“本许可证”授予您的专利许可自您发起专利诉讼或专利维权行动之日终止。
3. 无商标许可
“贡献者”在“本许可证”下不提供对其商品名称、商标、服务标识或产品名称的商标许可，但您为满足第4条规定的声明义务而必须使用的情形除外。
4. 分发限制
您可以将您接收到的“贡献”或您的“衍生作品”以源程序形式或可执行形式重新“分发”，但必须满足下列条件：
（1）您必须向接收者提供“本许可证”的副本，并保留“贡献”中的版权、商标、专利及免责声明；并且，
（2）如果您“分发”您接收到的“贡献”，您必须使用“本许可证”提供该“贡献”的源代码副本；如果您 “分发”您的“衍生作品”，您必须：
（i）随“衍生作品”提供使用“本许可证”“分发”的您的“衍生作品”的“对应源代码”。如果您通过下载链接提供前述“对应源代码”，则您应将下载链接地址置于“衍生作品”或其随附文档中的明显位置，有效期自该“衍生作品”“分发”之日起不少于三年，并确保接收者可以获得“对应源代码”；或者，
（ii）随“衍生作品”向接收者提供一个书面要约，表明您愿意提供根据“本许可证”“分发”的您“衍生作品”的“对应源代码”。该书面要约应置于“衍生作品”中的明显位置，并确保接收者根据书面要约可获取“对应源代码”的时间从您接到该请求之日起不得超过三个月，且有效期自该“衍生作品”“分发”之日起不少于三年。
5. 违约与终止
如果您违反“本许可证”，任何“贡献者”有权书面通知您终止其根据“本许可证”授予您的许可。该“贡献者”授予您的许可自您接到其终止通知之日起终止。仅在如下两种情形下，即使您收到“贡献者”的通知也并不终止其授予您的许可：
（1）您在接到该终止通知之前已停止所有违反行为；
（2）您是首次收到该“贡献者”根据“本许可证”发出的书面终止通知，并且您在收到该通知后30天内已停止所有违反行为。
只要您下游的接收者遵守“本许可证”的相关规定，即使您在“本许可证”下被授予的许可终止，不影响下游的接收者根据“本许可证”享有的权利。
6. 例外
如果您将“贡献”与采用GNU AFFERO GENERAL PUBLIC LICENSE Version 3（以下简称“AGPLv3”）或其后续版本的作品结合形成新的“衍生作品”，且根据“AGPLv3”或其后续版本的要求您有义务将新形成的“衍生作品”以“AGPLv3”或其后续版本进行许可的，您可以根据“AGPLv3”或其后续版本进行许可，只要您在“分发”该“衍生作品”的同时向接收者提供“本许可证”的副本，并保留“贡献”中的版权、商标、专利及免责声明。但任何“贡献者”不会因您选择“AGPLv3”或其后续版本而授予该“衍生作品”的接收者更多权利。
7. 免责声明与责任限制
“贡献”在提供时不带有任何明示或默示的担保。在任何情况下，“贡献者”或版权人不对任何人因使用“贡献”而引发的任何直接或间接损失承担任何责任，不论该等损失因何种原因导致或者基于何种法律理论,即使其曾被告知有该等损失的可能性。
8. 语言
“本许可证”以中英文双语表述，中英文版本具有同等法律效力。如果中英文版本存在任何不一致，以中文版为准。
条款结束
如何将木兰公共许可证，第2版，应用到您的软件
如果您希望将木兰公共许可证，第2版，应用到您的软件，为了方便接收者查阅，建议您完成如下三步：
1， 请您补充如下声明中的空白，包括软件名、软件的首次发表年份以及您作为版权人的名字；
2， 请您在软件包的一级目录下创建以“LICENSE”为名的文件，将整个许可证文本放入该文件中；
3， 请将如下声明文本放入每个源文件的头部注释中。
Copyright (c) 2021 OceanBase
OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
You can use this software according to the terms and conditions of the Mulan PubL v2.
You may obtain a copy of Mulan PubL v2 at:
http://license.coscl.org.cn/MulanPubL-2.0
THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
See the Mulan PubL v2 for more details.
Mulan Public License，Version 2
Mulan Public License，Version 2 (Mulan PubL v2)
May 2021 http://license.coscl.org.cn/MulanPubL-2.0
Your reproduction, use, modification and Distribution of the Contribution shall be subject to Mulan Public License, Version 2 (this License) with following terms and conditions:
0. Definition
Contribution means the copyrightable work licensed by a particular Contributor under this License, including the work licensed by the initial Contributor under this License and its Derivative Work licensed by any subsequent Contributor under this License.
Contributor means the Individual or Legal Entity who licenses its copyrightable work under this License.
Legal Entity means the entity making a Contribution and all its Affiliates.
Affiliates mmeans entities that control, are controlled by, or are under common control with the acting entity under this License, ‘control’ means direct or indirect ownership of at least fifty percent (50%) of the voting power, capital or other securities of controlled or commonly controlled entity.
Derivative Work means works created based on Contribution, specifically including works formed by modifying, rewriting, translating, annotating, combining or linking to all or part of Contribution (including dynamic linking or static linking). Works which only communicate with Contribution through inter-process communication or system call, are independent works, rather than Derivative Work.
Corresponding Source Code means all the source code needed to generate, install, and (for an executable work) run the object code including the interface definition files associated with source files for the work, and scripts to control those activities, excluding of compilation environment and compilation tools, cloud services platform (if any).
Distribute (or Distribution) means the act of making the Contribution or Derivative Work available to others through any medium, and using the Contribution or Derivative Work to provide online services to users, such as the act of providing online services through a cloud service platform built using Contributions or Derivative Works.
1. Grant of Copyright License
Subject to the terms and conditions of this License, each Contributor hereby grants to you a perpetual, worldwide, royalty-free, non-exclusive, irrevocable copyright license to reproduce, use, modify, or Distribute its Contribution or Derivative Work, with modification or not.
2. Grant of Patent License
Subject to the terms and conditions of this License, each Contributor hereby grants to you a perpetual, worldwide, royalty-free, non-exclusive, irrevocable (except for revocation under this Section) patent license to use, make, have made, sell, offer for sale, import or otherwise transfer its Contribution, where such patent license is only limited to the patent claims owned or controlled by such Contributor now or in future which will be necessarily infringed by its Contribution alone, excluding of any patent claims solely be infringed by your modification. If you or your Affiliates directly or indirectly institute patent litigation (including a cross claim or counterclaim in a litigation) or other patent enforcement activities against any individual or entity by alleging that any Contribution infringes patents, then any patent license granted to you under this License for the Contribution shall terminate as of the date such litigation or activity is filed or taken.
3. No Trademark License
No trademark license is granted to use the trade names, trademarks, service marks, or product names of Contributor, except as required to fulfill notice requirements in Section 4.
4. Distribution Restriction
You may Distribute the Contribution you received or your Derivative Work, whether in source or executable forms, provided that you meet the following conditions:
1) You must provide recipients with a copy of this License and retain copyright, trademark, patent and disclaimer statements in the Contribution; and,
2) If you Distribute the Contribution you received, you must provide copies of the Contribution’s source code under this License;
If you Distribute your Derivative Work, you have to:
(i) accompanying the Derivative work, provide recipients with Corresponding Source Code of your Derivative Work under this License. If you provide the Corresponding Source Code through a download link, you should place such link address prominently in the Derivative Work or its accompanying documents, and be valid no less than three years from your Distribution of the particular Derivative Work, and ensure that the recipients can acquire the Corresponding Source Code through the link; or,
(ii) accompanying the Derivative Work, provide recipients with a written offer indicating your willingness to provide the Corresponding Source Code of the Derivative Work licensed under this License. Such written offer shall be placed prominently in the Derivative Work or its accompanying documents. Without reasonable excuse, the recipient shall be able to acquire the Corresponding Source code of the Derivative work for no more than three months from your receipt of a valid request, and be valid no less than three years from your Distribution of the particular Derivative Work.
5. Breach and Termination
If you breach this License, any Contributor has the right to notify you in writing to terminate its license granted to you under this License. The license granted to you by such Contributor terminates upon your receipt of such notice of termination. Notwithstanding the foregoing, your license will not be terminated even if you receive a notice of termination from Contributor, provided that:
1) you have cured all the breaches prior to receiving such notice of termination; or,
2) it’s your first time to receive a notice of termination from such Contributor pursuant to this License, and you have cured all the breaches within 30 days of receipt of such notice.
Termination of your license under this License shall not affect the downstream recipient's rights under this License, provided that the downstream recipient complies with this License.
6. Exceptions
If you combine Contribution or your Derivative Work with a work licensed under the GNU AFFERO GENERAL PUBLIC LICENSE Version 3 (hereinafter referred to as “AGPLv3”) or its subsequent versions, and according to the AGPLv3 or its subsequent versions, you have an obligation to make the combined work to be licensed under the corresponding license, you can license such combined work under the license, provided that when you Distribute the combined work, you also provide a copy of this License to the recipients, and retain copyright, trademarks, patents, and disclaimer statements in the Contribution. No Contributor will grant additional rights to the recipients of the combined work for your license under AGPLv3 or its subsequent versions.
7. Disclaimer of Warranty and Limitation of liability
CONTRIBUTION ARE PROVIDED WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED. IN NO EVENT SHALL ANY CONTRIBUTOR OR COPYRIGHT HOLDER BE LIABLE TO YOU FOR ANY DAMAGES, INCLUDING, BUT NOT LIMITED TO ANY DIRECT, OR INDIRECT, SPECIAL OR CONSEQUENTIAL DAMAGES ARISING FROM YOUR USE OR INABILITY TO USE THE CONTRIBUTION, NO MATTER HOW IT’S CAUSED OR BASED ON WHICH LEGAL THEORY, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
8. Language
THIS LICENSE IS WRITTEN IN BOTH CHINESE AND ENGLISH, AND THE CHINESE VERSION AND ENGLISH VERSION SHALL HAVE THE SAME LEGAL EFFECT. IN THE CASE OF DIVERGENCE BETWEEN THE CHINESE AND ENGLISH VERSIONS, THE CHINESE VERSION SHALL PREVAIL.
END OF THE TERMS AND CONDITIONS
How to apply the Mulan Public License，Version 2 (Mulan PubL v2), to your software
To apply the Mulan Public License，Version 2 to your work, for easy identification by recipients, you are suggested to complete following three steps:
Fill in the blanks in following statement, including insert your software name, the year of the first publication of your software, and your name identified as the copyright owner;
Create a file named “LICENSE” which contains the whole context of this License in the first directory of your software package;
Attach the statement to the appropriate annotated syntax at the beginning of each source file.
Copyright (c) 2021 OceanBase
OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
You can use this software according to the terms and conditions of the Mulan PubL v2.
You may obtain a copy of Mulan PubL v2 at:
http://license.coscl.org.cn/MulanPubL-2.0
THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
See the Mulan PubL v2 for more details.

104
README.md Normal file
View File

@ -0,0 +1,104 @@
# OceanBase LogProxy
OceanBase LogProxy (CE) is a proxy service of [OceanBase CE](https://github.com/oceanbase/oceanbase). It is a part of [OMS](https://www.oceanbase.com/product/oms), and it can establish and manage connections with OceanBase for incremental log reading even with a isolated network.
## Instructions before use
### Version compatibility
LogProxy is based on [libobcdc](https://github.com/oceanbase/oceanbase/tree/master/src/logservice/libobcdc) (former `liboblog`), which is packaged in `oceanbase-ce-cdc` rpm file from `v4.0.0`. You can get it from the [software center](https://www.oceanbase.com/softwarecenter) or [official mirrors](https://mirrors.aliyun.com/oceanbase/community/stable/el/).
For `v1.1.0` or later versions of LogProxy, we no longer provide the rpm file for installation. Users should use the compressed package at [software center](https://www.oceanbase.com/softwarecenter) or [releases page](https://github.com/oceanbase/oblogproxy/releases), which already has a suitable `libobcdc` built in, so users no longer need to install `libobcdc` manually.
Please check the release note for detailed version compatibility information.
#### For OceanBase CE 3.x Users
The `libobcdc` or `liboblog` was packaged in `oceanbase-ce-devel` rpm file before `v4.0.0`, and users need to install it manually before deploy the LogProxy service.
The version correspondence between libobcdc and LogProxy is as follows.
| libobcdc | LogProxy |
|----------|----------|
| 3.1.1 | 1.0.0 |
| 3.1.2 | 1.0.1 |
| 3.1.3 | 1.0.2 |
| 3.1.4 | 1.0.3 |
### Installation
LogProxy service doesn't need params about OceanBase cluster to get started, one LogProxy can subscribe to multiple OceanBase clusters at the same time, and the connection configuration is passed from the client.
LogProxy will use a lot of memory, so it is strongly recommended to deploy it separately from the OceanBase server.
## Getting started
### Install
You can install a released version of LogProxy or build it from the source code.
#### Install a released version
If you want to install a released version, you can use the compressed package at the release page or software center. Here we take `/usr/local/oblogproxy` as the deployment directory.
```bash
wget https://github.com/oceanbase/oblogproxy/releases/download/v1.1.1/oblogproxy-ce-for-4x-1.1.1-20230418115957.tar.gz
tar -zxvf oblogproxy-ce-for-4x-1.1.1-20230418115957.tar.gz -C /usr/local
```
#### Build from source code
See [How to build](docs/build.md).
### Configure
For security reasons, LogProxy needs to configure the username and password of a certain user, which must be a sys tenant user of the OceanBase to connect with. Note that the username here should not contain cluster name or tenant name.
You can configure the username and password by one of the following ways:
- Add it to local conf at `conf/conf.json`.
- Set it in the client params. See the [client doc](https://github.com/oceanbase/oblogclient/blob/master/docs/quickstart/logproxy-client-tutorial.md) for details.
#### Add it to local conf
Firstly, set the libobcdc path to environment, here we use the builtin libobcdc in the deployment directory.
```bash
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/oblogproxy/libobcdc
```
Get the encrypted username and password.
```bash
./bin/logproxy -x username
./bin/logproxy -x password
```
Then add the outputs to `ob_sys_username` and `ob_sys_password` at `conf/conf.json`.
### Start
You can start the service by the following command.
```bash
bash ./run.sh start
```
Then you can use [oblogclient](https://github.com/oceanbase/oblogclient) to subscribe the log data from LogProxy, and the service is bind to port `2983` by default.
The service log of LogProxy is located at `logs/`, and the service log of LogReader (task thread) is located at `run/{client-id}/logs/`.
## Licencing
OceanBase Database is under MulanPubL - 2.0 license. You can freely copy and use the source code. When you modify or distribute the source code, please obey the MulanPubL - 2.0 license.
## Contributing
Contributions are warmly welcomed and greatly appreciated. Here are a few ways you can contribute:
- Raise us an [issue](https://github.com/oceanbase/oblogproxy/issues).
- Submit Pull Requests.
## Support
In case you have any problems when using OceanBase LogProxy, welcome reach out for help:
- [GitHub Issue](https://github.com/oceanbase/oblogproxy/issues)
- [Official Website](https://open.oceanbase.com/)

53
cmake/gflags.cmake Normal file
View File

@ -0,0 +1,53 @@
INCLUDE(ExternalProject)
if (POLICY CMP0097)
CMAKE_POLICY(SET CMP0097 NEW)
endif ()
SET(GFLAGS_SOURCES_DIR ${THIRD_PARTY_PATH}/gflags)
SET(GFLAGS_INSTALL_DIR ${THIRD_PARTY_PATH}/install/gflags)
SET(GFLAGS_INCLUDE_DIR "${GFLAGS_INSTALL_DIR}/include" CACHE PATH "gflags include directory." FORCE)
set(GFLAGS_LIBRARIES "${GFLAGS_INSTALL_DIR}/lib/libgflags.a" CACHE FILEPATH "GFLAGS_LIBRARIES" FORCE)
set(BUILD_COMMAND $(MAKE) --silent)
set(INSTALL_COMMAND $(MAKE) install)
INCLUDE_DIRECTORIES(${GFLAGS_INCLUDE_DIR})
ExternalProject_Add(
extern_gflags
${EXTERNAL_PROJECT_LOG_ARGS}
GIT_REPOSITORY "https://github.com/gflags/gflags.git"
GIT_TAG "v2.2.2"
GIT_SUBMODULES ""
GIT_SUBMODULES_RECURSE "false"
PREFIX ${GFLAGS_SOURCES_DIR}
BUILD_COMMAND ${BUILD_COMMAND}
INSTALL_COMMAND ${INSTALL_COMMAND}
UPDATE_COMMAND ""
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
-DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE}
-DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}
-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
-DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG}
-DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE}
-DBUILD_STATIC_LIBS=ON
-DCMAKE_INSTALL_PREFIX=${GFLAGS_INSTALL_DIR}
-DCMAKE_INSTALL_LIBDIR=${GFLAGS_INSTALL_DIR}/lib
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DGFLAGS_BUILD_TESTING=OFF
-DGFLAGS_BUILD_PACKAGING=OFF
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
${EXTERNAL_OPTIONAL_ARGS}
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GFLAGS_INSTALL_DIR}
-DCMAKE_INSTALL_LIBDIR:PATH=${GFLAGS_INSTALL_DIR}/lib
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
-DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
)
ADD_LIBRARY(gflags STATIC IMPORTED GLOBAL)
SET_PROPERTY(TARGET gflags PROPERTY IMPORTED_LOCATION ${GFLAGS_LIBRARIES})
ADD_DEPENDENCIES(gflags extern_gflags)
message(gflag: ${GFLAGS_INCLUDE_DIR}, ${GFLAGS_LIBRARIES})

53
cmake/glog.cmake Normal file
View File

@ -0,0 +1,53 @@
INCLUDE(ExternalProject)
SET(GLOG_SOURCES_DIR ${THIRD_PARTY_PATH}/glog)
SET(GLOG_INSTALL_DIR ${THIRD_PARTY_PATH}/install/glog)
SET(GLOG_INCLUDE_DIR "${GLOG_INSTALL_DIR}/include" CACHE PATH "glog include directory." FORCE)
SET(GLOG_LIBRARIES "${GLOG_INSTALL_DIR}/lib/libglog.a" CACHE FILEPATH "glog library." FORCE)
INCLUDE_DIRECTORIES(${GLOG_INCLUDE_DIR})
set(prefix_path "${THIRD_PARTY_PATH}/install/gflags")
SET(gflags_BUILD_STATIC_LIBS ON)
ExternalProject_Add(
extern_glog
${EXTERNAL_PROJECT_LOG_ARGS}
DEPENDS gflags
GIT_REPOSITORY "https://github.com/google/glog.git"
GIT_TAG "v0.5.0"
GIT_SUBMODULES ""
GIT_SUBMODULES_RECURSE "false"
PREFIX ${GLOG_SOURCES_DIR}
UPDATE_COMMAND ""
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
-DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE}
-DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}
-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
-DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG}
-DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE}
-DCMAKE_INSTALL_PREFIX=${GLOG_INSTALL_DIR}
-DCMAKE_INSTALL_LIBDIR=${GLOG_INSTALL_DIR}/lib
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DBUILD_SHARED_LIBS=OFF
-DWITH_GFLAGS=ON
-DWITH_GTEST=OFF
-Dgflags_DIR=${GFLAGS_INSTALL_DIR}/lib/cmake/gflags
-DBUILD_TESTING=OFF
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
-DCMAKE_PREFIX_PATH=${prefix_path}
${EXTERNAL_OPTIONAL_ARGS}
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GLOG_INSTALL_DIR}
-DCMAKE_INSTALL_LIBDIR:PATH=${GLOG_INSTALL_DIR}/lib
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
-DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
)
ADD_DEPENDENCIES(extern_glog gflags)
ADD_LIBRARY(glog STATIC IMPORTED GLOBAL)
SET_PROPERTY(TARGET glog PROPERTY IMPORTED_LOCATION ${GLOG_LIBRARIES})
ADD_DEPENDENCIES(glog extern_glog)
LINK_LIBRARIES(glog gflags)

46
cmake/gtest.cmake Normal file
View File

@ -0,0 +1,46 @@
INCLUDE(ExternalProject)
SET(GTEST_SOURCES_DIR ${THIRD_PARTY_PATH}/gtest)
SET(GTEST_INSTALL_DIR ${THIRD_PARTY_PATH}/install/gtest)
SET(GTEST_INCLUDE_DIR "${GTEST_INSTALL_DIR}/include" CACHE PATH "gtest include directory." FORCE)
SET(GTEST_LIBRARIES "${GTEST_INSTALL_DIR}/lib/libgtest.a" CACHE FILEPATH "gtest library." FORCE)
set(INSTALL_COMMAND $(MAKE) install)
INCLUDE_DIRECTORIES(${GTEST_INCLUDE_DIR})
set(prefix_path "${THIRD_PARTY_PATH}/install/gflags")
SET(gflags_BUILD_STATIC_LIBS ON)
ExternalProject_Add(
extern_gtest
${EXTERNAL_PROJECT_LOG_ARGS}
DEPENDS gflags
GIT_REPOSITORY "https://github.com/google/googletest.git"
GIT_TAG "release-1.11.0"
PREFIX ${GTEST_SOURCES_DIR}
UPDATE_COMMAND ""
INSTALL_COMMAND ${INSTALL_COMMAND}
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
-DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE}
-DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}
-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
-DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG}
-DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE}
-DCMAKE_INSTALL_PREFIX=${GTEST_INSTALL_DIR}
-DCMAKE_INSTALL_LIBDIR=${GTEST_INSTALL_DIR}/lib
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DBUILD_GMOCK=OFF
-Dgtest_build_samples=OFF
-Dgtest_build_tests=OFF
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
-DCMAKE_PREFIX_PATH=${prefix_path}
${EXTERNAL_OPTIONAL_ARGS}
)
ADD_DEPENDENCIES(extern_gtest gflags)
ADD_LIBRARY(gtest STATIC IMPORTED GLOBAL)
SET_PROPERTY(TARGET gtest PROPERTY IMPORTED_LOCATION ${GTEST_LIBRARIES})
ADD_DEPENDENCIES(gtest extern_gtest)
LINK_LIBRARIES(gtest gflags)

39
cmake/jsoncpp.cmake Normal file
View File

@ -0,0 +1,39 @@
INCLUDE(ExternalProject)
SET(JSONCPP_SOURCES_DIR ${THIRD_PARTY_PATH}/jsoncpp)
SET(JSONCPP_INSTALL_DIR ${THIRD_PARTY_PATH}/install/jsoncpp)
SET(JSONCPP_INCLUDE_DIR "${JSONCPP_INSTALL_DIR}/include" CACHE PATH "jsoncpp include directory." FORCE)
SET(JSONCPP_LIBRARIES "${JSONCPP_INSTALL_DIR}/lib/libjsoncpp.a" CACHE FILEPATH "jsoncpp library." FORCE)
INCLUDE_DIRECTORIES(${JSONCPP_INCLUDE_DIR})
set(prefix_path "${THIRD_PARTY_PATH}/install/jsoncpp")
ExternalProject_Add(
extern_jsoncpp
${EXTERNAL_PROJECT_LOG_ARGS}
GIT_REPOSITORY "https://github.com/open-source-parsers/jsoncpp.git"
GIT_TAG "1.9.0"
PREFIX ${JSONCPP_SOURCES_DIR}
UPDATE_COMMAND ""
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
-DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE}
-DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}
-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
-DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG}
-DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE}
-DCMAKE_INSTALL_PREFIX=${JSONCPP_INSTALL_DIR}
-DCMAKE_INSTALL_LIBDIR=${JSONCPP_INSTALL_DIR}/lib
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DJSONCPP_WITH_TESTS=OFF
-DJSONCPP_WITH_POST_BUILD_UNITTEST=OFF
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
-DCMAKE_PREFIX_PATH=${prefix_path}
${EXTERNAL_OPTIONAL_ARGS}
)
ADD_LIBRARY(jsoncpp STATIC IMPORTED GLOBAL)
SET_PROPERTY(TARGET jsoncpp PROPERTY IMPORTED_LOCATION ${JSONCPP_LIBRARIES})
ADD_DEPENDENCIES(jsoncpp extern_jsoncpp)

46
cmake/libevent.cmake Normal file
View File

@ -0,0 +1,46 @@
INCLUDE(ExternalProject)
SET(MODULE_NAME libevent)
SET(LIBEVENT_SOURCES_DIR ${THIRD_PARTY_PATH}/${MODULE_NAME})
SET(LIBEVENT_INSTALL_DIR ${THIRD_PARTY_PATH}/install/${MODULE_NAME})
SET(LIBEVENT_INCLUDE_DIR "${LIBEVENT_INSTALL_DIR}/include" CACHE PATH "libevent include directory." FORCE)
SET(LIBEVENT_LIBRARIES "${LIBEVENT_INSTALL_DIR}/lib/libevent.a" CACHE FILEPATH "libevent library." FORCE)
INCLUDE_DIRECTORIES(${LIBEVENT_INCLUDE_DIR})
set(prefix_path "${LIBEVENT_INSTALL_DIR}")
ExternalProject_Add(
extern_libevent
${EXTERNAL_PROJECT_LOG_ARGS}
GIT_REPOSITORY "https://github.com/libevent/libevent"
GIT_TAG "release-2.1.12-stable"
PREFIX ${LIBEVENT_SOURCES_DIR}
UPDATE_COMMAND ""
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
-DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE}
-DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}
-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
-DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG}
-DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE}
-DCMAKE_INSTALL_PREFIX=${LIBEVENT_INSTALL_DIR}
-DCMAKE_INSTALL_LIBDIR=${LIBEVENT_INSTALL_DIR}/lib
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DEVENT__DISABLE_OPENSSL=ON
-DEVENT__DISABLE_TESTS=ON
-DEVENT__DISABLE_DEBUG_MODE=ON
-DEVENT__DISABLE_BENCHMARK=ON
-DEVENT__DISABLE_REGRESS=ON
-DEVENT__DISABLE_SAMPLES=ON
-DEVENT__FORCE_KQUEUE_CHECK=ON
-DEVENT__LIBRARY_TYPE=STATIC
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
-DCMAKE_PREFIX_PATH=${prefix_path}
${EXTERNAL_OPTIONAL_ARGS}
)
ADD_LIBRARY(${MODULE_NAME} STATIC IMPORTED GLOBAL)
SET_PROPERTY(TARGET ${MODULE_NAME} PROPERTY IMPORTED_LOCATION ${LIBEVENT_LIBRARIES})
ADD_DEPENDENCIES(${MODULE_NAME} extern_libevent)

37
cmake/lz4.cmake Normal file
View File

@ -0,0 +1,37 @@
INCLUDE(ExternalProject)
SET(LZ4_SOURCES_DIR ${THIRD_PARTY_PATH}/lz4)
SET(LZ4_INSTALL_DIR ${THIRD_PARTY_PATH}/install/lz4)
SET(LZ4_INCLUDE_DIR "${LZ4_INSTALL_DIR}/lib" CACHE PATH "lz4 include directory." FORCE)
SET(LZ4_LIBRARIES "${LZ4_INSTALL_DIR}/lib/liblz4.a" CACHE FILEPATH "lz4 library." FORCE)
FILE(WRITE ${LZ4_SOURCES_DIR}/src/build.sh
"make clean;CFLAGS=-fPIC CXXFLAGS=-fPIC make VERBOSE=1 -j${NUM_OF_PROCESSOR} liblz4.a"
)
INCLUDE_DIRECTORIES(${LZ4_INCLUDE_DIR})
set(prefix_path "${THIRD_PARTY_PATH}/install/lz4")
ExternalProject_Add(
extern_lz4
${EXTERNAL_PROJECT_LOG_ARGS}
GIT_REPOSITORY "https://github.com/lz4/lz4.git"
GIT_TAG "v1.9.4"
PREFIX ${LZ4_SOURCES_DIR}
BUILD_IN_SOURCE ON
UPDATE_COMMAND ""
CONFIGURE_COMMAND ""
BUILD_COMMAND mv ../build.sh . COMMAND sh build.sh
INSTALL_COMMAND mkdir -p ${LZ4_INSTALL_DIR} COMMAND cp -r ${LZ4_SOURCES_DIR}/src/extern_lz4/lib ${LZ4_INSTALL_DIR}/
${EXTERNAL_OPTIONAL_ARGS}
CMAKE_ARGS
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
)
ADD_LIBRARY(lz4 STATIC IMPORTED GLOBAL)
SET_PROPERTY(TARGET lz4 PROPERTY IMPORTED_LOCATION ${LZ4_LIBRARIES})
ADD_DEPENDENCIES(lz4 extern_lz4)

36
cmake/oblogmsg.cmake Normal file
View File

@ -0,0 +1,36 @@
INCLUDE(ExternalProject)
if (POLICY CMP0097)
cmake_policy(SET CMP0097 NEW)
endif ()
SET(OBLOGMSG_SOURCES_DIR ${THIRD_PARTY_PATH}/oblogmsg)
SET(OBLOGMSG_DOWNLOAD_DIR "${OBLOGMSG_SOURCES_DIR}/src/extern_oblogmsg")
SET(OBLOGMSG_INSTALL_DIR ${THIRD_PARTY_PATH}/install/oblogmsg)
SET(OBLOGMSG_INCLUDE_DIR "${OBLOGMSG_INSTALL_DIR}/include;${THIRD_PARTY_PATH}/install/oblogmsg/drcmessage" CACHE PATH "oblogmsg include directory." FORCE)
SET(OBLOGMSG_LIB_DIR "${OBLOGMSG_INSTALL_DIR}/lib/" CACHE FILEPATH "oblogmsg library directory." FORCE)
SET(OBLOGMSG_LIBRARIES "liboblogmsg.a" CACHE FILEPATH "oblogmsg library." FORCE)
ExternalProject_Add(
extern_oblogmsg
${EXTERNAL_PROJECT_LOG_ARGS}
GIT_REPOSITORY "https://github.com/oceanbase/oblogmsg"
GIT_TAG "master"
GIT_SUBMODULES ""
GIT_SUBMODULES_RECURSE "false"
PREFIX ${OBLOGMSG_SOURCES_DIR}
BUILD_IN_SOURCE 1
UPDATE_COMMAND ""
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DCMAKE_BUILD_TYPE=Debug
-DCMAKE_PREFIX_PATH=${prefix_path}
INSTALL_COMMAND mkdir -p ${OBLOGMSG_INSTALL_DIR}/lib ${OBLOGMSG_INSTALL_DIR}/include COMMAND cp -r ${OBLOGMSG_SOURCES_DIR}/src/extern_oblogmsg/src/${OBLOGMSG_LIBRARIES} ${OBLOGMSG_INSTALL_DIR}/lib/ COMMAND cp -r ${OBLOGMSG_SOURCES_DIR}/src/extern_oblogmsg/include ${OBLOGMSG_INSTALL_DIR}/ COMMAND cp -r ${OBLOGMSG_SOURCES_DIR}/src/extern_oblogmsg/src/${OBLOGMSG_LIBRARIES} ${LIBRARY_OUTPUT_PATH}/
)
ADD_LIBRARY(oblogmsg STATIC IMPORTED GLOBAL)
SET_PROPERTY(TARGET oblogmsg PROPERTY IMPORTED_LOCATION ${OBLOGMSG_LIBRARIES})
ADD_DEPENDENCIES(oblogmsg extern_oblogmsg)

156
cmake/protobuf.cmake Normal file
View File

@ -0,0 +1,156 @@
INCLUDE(ExternalProject)
# Always invoke `FIND_PACKAGE(Protobuf)` for importing function protobuf_generate_cpp
FIND_PACKAGE(Protobuf QUIET)
macro(UNSET_VAR VAR_NAME)
UNSET(${VAR_NAME} CACHE)
UNSET(${VAR_NAME})
endmacro()
UNSET_VAR(PROTOBUF_INCLUDE_DIR)
UNSET_VAR(PROTOBUF_FOUND)
UNSET_VAR(PROTOBUF_PROTOC_EXECUTABLE)
UNSET_VAR(PROTOBUF_PROTOC_LIBRARY)
UNSET_VAR(PROTOBUF_LITE_LIBRARY)
UNSET_VAR(PROTOBUF_LIBRARY)
UNSET_VAR(PROTOBUF_INCLUDE_DIR)
UNSET_VAR(Protobuf_PROTOC_EXECUTABLE)
if (POLICY CMP0097)
cmake_policy(SET CMP0097 NEW)
endif ()
# Print and set the protobuf library information,
# finish this cmake process and exit from this file.
macro(PROMPT_PROTOBUF_LIB)
SET(protobuf_DEPS ${ARGN})
MESSAGE(STATUS "Protobuf protoc executable: ${PROTOBUF_PROTOC_EXECUTABLE}")
MESSAGE(STATUS "Protobuf-lite library: ${PROTOBUF_LITE_LIBRARY}")
MESSAGE(STATUS "Protobuf library: ${PROTOBUF_LIBRARY}")
MESSAGE(STATUS "Protoc library: ${PROTOBUF_PROTOC_LIBRARY}")
MESSAGE(STATUS "Protobuf version: ${PROTOBUF_VERSION}")
INCLUDE_DIRECTORIES(${PROTOBUF_INCLUDE_DIR})
# Assuming that all the protobuf libraries are of the same type.
IF (${PROTOBUF_LIBRARY} MATCHES ${CMAKE_STATIC_LIBRARY_SUFFIX})
SET(protobuf_LIBTYPE STATIC)
ELSEIF (${PROTOBUF_LIBRARY} MATCHES "${CMAKE_SHARED_LIBRARY_SUFFIX}$")
SET(protobuf_LIBTYPE SHARED)
ELSE ()
MESSAGE(FATAL_ERROR "Unknown library type: ${PROTOBUF_LIBRARY}")
ENDIF ()
ADD_LIBRARY(protobuf ${protobuf_LIBTYPE} IMPORTED GLOBAL)
SET_PROPERTY(TARGET protobuf PROPERTY IMPORTED_LOCATION ${PROTOBUF_LIBRARY})
ADD_LIBRARY(protobuf_lite ${protobuf_LIBTYPE} IMPORTED GLOBAL)
SET_PROPERTY(TARGET protobuf_lite PROPERTY IMPORTED_LOCATION ${PROTOBUF_LITE_LIBRARY})
ADD_LIBRARY(libprotoc ${protobuf_LIBTYPE} IMPORTED GLOBAL)
SET_PROPERTY(TARGET libprotoc PROPERTY IMPORTED_LOCATION ${PROTOC_LIBRARY})
ADD_EXECUTABLE(protoc IMPORTED GLOBAL)
SET_PROPERTY(TARGET protoc PROPERTY IMPORTED_LOCATION ${PROTOBUF_PROTOC_EXECUTABLE})
SET(Protobuf_PROTOC_EXECUTABLE ${PROTOBUF_PROTOC_EXECUTABLE})
FOREACH (dep ${protobuf_DEPS})
ADD_DEPENDENCIES(protobuf ${dep})
ADD_DEPENDENCIES(protobuf_lite ${dep})
ADD_DEPENDENCIES(libprotoc ${dep})
ADD_DEPENDENCIES(protoc ${dep})
ENDFOREACH ()
RETURN()
endmacro()
macro(SET_PROTOBUF_VERSION)
EXEC_PROGRAM(${PROTOBUF_PROTOC_EXECUTABLE} ARGS --version OUTPUT_VARIABLE PROTOBUF_VERSION)
STRING(REGEX MATCH "[0-9]+.[0-9]+" PROTOBUF_VERSION "${PROTOBUF_VERSION}")
endmacro()
FUNCTION(build_protobuf TARGET_NAME)
STRING(REPLACE "extern_" "" TARGET_DIR_NAME "${TARGET_NAME}")
SET(PROTOBUF_SOURCES_DIR ${THIRD_PARTY_PATH}/${TARGET_DIR_NAME})
SET(PROTOBUF_INSTALL_DIR ${THIRD_PARTY_PATH}/install/${TARGET_DIR_NAME})
SET(${TARGET_NAME}_INCLUDE_DIR "${PROTOBUF_INSTALL_DIR}/include" PARENT_SCOPE)
SET(PROTOBUF_INCLUDE_DIR "${PROTOBUF_INSTALL_DIR}/include" PARENT_SCOPE)
SET(${TARGET_NAME}_LITE_LIBRARY
"${PROTOBUF_INSTALL_DIR}/lib/libprotobuf-lite${CMAKE_STATIC_LIBRARY_SUFFIX}"
PARENT_SCOPE)
SET(${TARGET_NAME}_LIBRARY
"${PROTOBUF_INSTALL_DIR}/lib/libprotobuf${CMAKE_STATIC_LIBRARY_SUFFIX}"
PARENT_SCOPE)
SET(${TARGET_NAME}_PROTOC_LIBRARY
"${PROTOBUF_INSTALL_DIR}/lib/libprotoc${CMAKE_STATIC_LIBRARY_SUFFIX}"
PARENT_SCOPE)
SET(${TARGET_NAME}_PROTOC_EXECUTABLE
"${PROTOBUF_INSTALL_DIR}/bin/protoc${CMAKE_EXECUTABLE_SUFFIX}"
PARENT_SCOPE)
SET(OPTIONAL_CACHE_ARGS "")
SET(OPTIONAL_ARGS
"-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
"-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
"-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}"
"-DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG}"
"-DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE}"
"-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}"
"-DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE}"
"-DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}"
"-DCMAKE_EXE_LINKER_FLAGS=${PROTOC_LINK_FLAGS}"
"-Dprotobuf_WITH_ZLIB=OFF"
${EXTERNAL_OPTIONAL_ARGS})
SET(PROTOBUF_REPO "https://github.com/protocolbuffers/protobuf.git")
SET(PROTOBUF_TAG "v3.6.0")
ExternalProject_Add(
${TARGET_NAME}
${EXTERNAL_PROJECT_LOG_ARGS}
PREFIX ${PROTOBUF_SOURCES_DIR}
UPDATE_COMMAND ""
GIT_REPOSITORY ${PROTOBUF_REPO}
GIT_TAG ${PROTOBUF_TAG}
GIT_SUBMODULES ""
GIT_SUBMODULES_RECURSE "false"
CONFIGURE_COMMAND
${CMAKE_COMMAND} ${PROTOBUF_SOURCES_DIR}/src/${TARGET_NAME}/cmake
${OPTIONAL_ARGS}
-Dprotobuf_BUILD_TESTS=OFF
-DCMAKE_SKIP_RPATH=ON
-Dprotobuf_VERBOSE=ON
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
-DCMAKE_INSTALL_PREFIX=${PROTOBUF_INSTALL_DIR}
-DCMAKE_INSTALL_LIBDIR=lib
-DBUILD_SHARED_LIBS=OFF
-Dprotobuf_MSVC_STATIC_RUNTIME=${MSVC_STATIC_CRT}
CMAKE_CACHE_ARGS
-DCMAKE_INSTALL_PREFIX:PATH=${PROTOBUF_INSTALL_DIR}
-DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
-DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
${OPTIONAL_CACHE_ARGS}
)
ENDFUNCTION()
SET(PROTOBUF_VERSION 3.6.0)
build_protobuf(extern_protobuf FALSE)
SET(PROTOBUF_INCLUDE_DIR ${extern_protobuf_INCLUDE_DIR}
CACHE PATH "protobuf include directory." FORCE)
SET(PROTOBUF_LITE_LIBRARY ${extern_protobuf_LITE_LIBRARY}
CACHE FILEPATH "protobuf lite library." FORCE)
SET(PROTOBUF_LIBRARY ${extern_protobuf_LIBRARY}
CACHE FILEPATH "protobuf library." FORCE)
SET(PROTOBUF_LIBRARIES ${extern_protobuf_LIBRARY}
CACHE FILEPATH "protobuf library." FORCE)
SET(PROTOBUF_PROTOC_LIBRARY ${extern_protobuf_PROTOC_LIBRARY}
CACHE FILEPATH "protoc library." FORCE)
SET(PROTOBUF_PROTOC_EXECUTABLE ${extern_protobuf_PROTOC_EXECUTABLE}
CACHE FILEPATH "protobuf executable." FORCE)
PROMPT_PROTOBUF_LIB(extern_protobuf)

31
cmake/rocksdb.cmake Normal file
View File

@ -0,0 +1,31 @@
INCLUDE(ExternalProject)
SET(ROCKSDB_SOURCES_DIR ${THIRD_PARTY_PATH}/rocksdb)
SET(ROCKSDB_INSTALL_DIR ${THIRD_PARTY_PATH}/install/rocksdb)
SET(ROCKSDB_INCLUDE_DIR "${ROCKSDB_INSTALL_DIR}/include" CACHE PATH "rocksdb include directory." FORCE)
SET(ROCKSDB_LIBRARIES "${ROCKSDB_INSTALL_DIR}/lib/librocksdb.a" CACHE FILEPATH "rocksdb library." FORCE)
INCLUDE_DIRECTORIES(${ROCKSDB_INCLUDE_DIR})
FILE(WRITE ${ROCKSDB_SOURCES_DIR}/src/build.sh
"PORTABLE=1 make -j${NUM_OF_PROCESSOR} static_lib"
)
ExternalProject_Add(
extern_rocksdb
${EXTERNAL_PROJECT_LOG_ARGS}
DEPENDS gflags zlib snappy bz2
PREFIX ${ROCKSDB_SOURCES_DIR}
GIT_REPOSITORY "https://github.com/facebook/rocksdb.git"
GIT_TAG "v6.3.6"
UPDATE_COMMAND ""
CONFIGURE_COMMAND ""
BUILD_IN_SOURCE 1
BUILD_COMMAND mv ../build.sh . COMMAND sh build.sh
INSTALL_COMMAND mkdir -p ${ROCKSDB_INSTALL_DIR}/lib COMMAND cp -r include ${ROCKSDB_INSTALL_DIR}/ COMMAND cp librocksdb.a ${ROCKSDB_LIBRARIES}
)
ADD_DEPENDENCIES(extern_rocksdb lz4 gflags)
ADD_LIBRARY(rocksdb STATIC IMPORTED GLOBAL)
SET_PROPERTY(TARGET rocksdb PROPERTY IMPORTED_LOCATION ${ROCKSDB_LIBRARIES})
ADD_DEPENDENCIES(rocksdb extern_rocksdb)
LINK_LIBRARIES(rocksdb gflags lz4)

110
cmake/rpm.cmake Normal file
View File

@ -0,0 +1,110 @@
set(CPACK_GENERATOR "RPM")
# use seperated RPM SPECs and generate different RPMs
set(CPACK_COMPONENTS_IGNORE_GROUPS 1)
set(CPACK_RPM_COMPONENT_INSTALL ON)
# use "oblogproxy" as main component so its RPM filename won't have "oblogproxy"
set(CPACK_RPM_MAIN_COMPONENT "oblogproxy")
# let rpmbuild determine rpm filename
set(CPACK_RPM_FILE_NAME "RPM-DEFAULT")
set(CPACK_RPM_PACKAGE_RELEASE ${OBLOGPROXY_RELEASEID})
set(CPACK_RPM_PACKAGE_RELEASE_DIST ON)
# RPM package informations.
set(CPACK_PACKAGING_INSTALL_PREFIX /usr/local/oblogproxy)
list(APPEND CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "/usr/local/oblogproxy")
set(CPACK_PACKAGE_NAME ${OBLOGPROXY_PACKAGE_NAME})
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "oblogproxy is a clog proxy server for OceanBase CE")
set(CPACK_PACKAGE_VENDOR "Ant Group CO., Ltd.")
set(CPACK_PACKAGE_VERSION ${OBLOGPROXY_PACKAGE_VERSION})
set(CPACK_PACKAGE_VERSION_MAJOR 1)
set(CPACK_PACKAGE_VERSION_MINOR 0)
set(CPACK_PACKAGE_VERSION_PATCH 1)
set(CPACK_RPM_PACKAGE_GROUP "Applications/Databases")
set(CPACK_RPM_PACKAGE_URL "https://open.oceanbase.com")
set(CPACK_RPM_PACKAGE_DESCRIPTION "oblogproxy is a clog proxy server for OceanBase CE")
set(CPACK_RPM_PACKAGE_LICENSE "Mulan PubL v2.")
set(CPACK_RPM_DEFAULT_USER "root")
set(CPACK_RPM_DEFAULT_GROUP "root")
set(CPACK_RPM_SPEC_MORE_DEFINE
"%global _missing_build_ids_terminate_build 0
%global _find_debuginfo_opts -g
%define __debug_install_post %{_rpmconfigdir}/find-debuginfo.sh %{?_find_debuginfo_opts} %{_builddir}/%{?buildsubdir};%{nil}
%define debug_package %{nil}")
## TIPS
#
# - PATH is relative to the **ROOT directory** of project other than the cmake directory.
if (NOT ${OBLOGPROXY_INSTALL_PREFIX})
set(CPACK_PACKAGING_INSTALL_PREFIX ${OBLOGPROXY_INSTALL_PREFIX})
endif()
message("CPACK_PACKAGING_INSTALL_PREFIX: ${CPACK_PACKAGING_INSTALL_PREFIX}")
list(APPEND OBLOGPROXY_BIN_FILES ${OMS_PROJECT_BUILD_PATH}/logproxy)
list(APPEND OBLOGPROXY_CONF_FILES ${CMAKE_SOURCE_DIR}/conf/conf.json)
list(APPEND OBLOGPROXY_SCRIPT_FILES ${CMAKE_SOURCE_DIR}/script/run.sh)
install(PROGRAMS
${OBLOGPROXY_BIN_FILES}
DESTINATION ${CPACK_PACKAGING_INSTALL_PREFIX}/bin
COMPONENT oblogproxy
)
install(FILES
${OBLOGPROXY_CONF_FILES}
DESTINATION ${CPACK_PACKAGING_INSTALL_PREFIX}/conf
COMPONENT oblogproxy
)
install(PROGRAMS
${OBLOGPROXY_SCRIPT_FILES}
DESTINATION ${CPACK_PACKAGING_INSTALL_PREFIX}/
COMPONENT oblogproxy
)
if (WITH_DEPS)
if (NOT USE_LIBOBLOG)
if (USE_LIBOBLOG_3)
file(GLOB LIBOBLOG_OUTPUT_LIBS
${DEP_VAR}/usr/lib/${OBCDC_NAME}*.so*
${DEP_VAR}/usr/local/oceanbase/deps/devel/lib/libaio.so*
)
install(FILES
${LIBOBLOG_OUTPUT_LIBS}
DESTINATION ${CPACK_PACKAGING_INSTALL_PREFIX}/liboblog
COMPONENT oblogproxy
)
else()
file(GLOB LIBOBLOG_OUTPUT_LIBS ${DEP_VAR}/home/admin/oceanbase/lib64/${OBCDC_NAME}*.so*
${DEP_VAR}/usr/local/oceanbase/deps/devel/lib/libaio.so*
${DEP_VAR}/usr/local/oceanbase/deps/devel/lib/mariadb/libmariadb.so*
)
install(FILES
${LIBOBLOG_OUTPUT_LIBS}
DESTINATION ${CPACK_PACKAGING_INSTALL_PREFIX}/liboblog
COMPONENT oblogproxy
)
endif()
endif()
endif()
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/utils_post.script "/sbin/ldconfig /usr/lib")
set(CPACK_RPM_UTILS_POST_INSTALL_SCRIPT_FILE ${CMAKE_CURRENT_BINARY_DIR}/utils_post.script)
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/utils_postun.script "/sbin/ldconfig")
set(CPACK_RPM_UTILS_POST_UNINSTALL_SCRIPT_FILE ${CMAKE_CURRENT_BINARY_DIR}/utils_postun.script)
if(USE_OBCDC_NS)
if (USE_LIBOBLOG_3)
set(CPACK_RPM_PACKAGE_REQUIRES "devdeps-libaio >= 0.3.112")
else()
set(CPACK_RPM_PACKAGE_REQUIRES "devdeps-libaio >= 0.3.112")
endif()
else()
set(CPACK_RPM_PACKAGE_REQUIRES "devdeps-libaio >= 0.3.112, oceanbase-ce-devel = 3.1.2")
endif()
# install cpack to make everything work
include(CPack)
#add rpm target to create RPMS
add_custom_target(rpm
COMMAND +make package
DEPENDS
logproxy)

41
conf/conf.json Normal file
View File

@ -0,0 +1,41 @@
{
"service_port": 2983,
"encode_threadpool_size": 8,
"encode_queue_size": 20000,
"max_packet_bytes": 67108864,
"record_queue_size": 20000,
"read_timeout_us": 2000000,
"read_fail_interval_us": 1000000,
"read_wait_num": 20000,
"send_timeout_us": 2000000,
"send_fail_interval_us": 1000000,
"check_quota_enable": false,
"command_timeout_s": 10,
"log_quota_size_mb": 5120,
"log_quota_day": 7,
"log_gc_interval_s": 43200,
"oblogreader_path_retain_hour": 168,
"oblogreader_lease_s": 300,
"oblogreader_path": "./run",
"allow_all_tenant": true,
"auth_user": true,
"auth_use_rs": false,
"auth_allow_sys_user": true,
"ob_sys_username": "",
"ob_sys_password": "",
"counter_interval_s": 2,
"metric_enable": true,
"metric_interval_s": 10,
"debug": false,
"verbose": false,
"verbose_packet": false,
"readonly": false,
"count_record": false,
"channel_type": "plain",
"tls_ca_cert_file": "",
"tls_cert_file": "",
"tls_key_file": "",
"tls_verify_peer": true,
"liboblog_tls": false,
"liboblog_tls_cert_path": ""
}

166
deps/dep_create.sh vendored Normal file
View File

@ -0,0 +1,166 @@
#!/bin/bash
#clear env
unalias -a
PWD="$(cd $(dirname $0); pwd)"
TARGET_DIR=${PWD}
if [[ ! -z "$1" ]]; then
TARGET_DIR=$1
mkdir -p ${TARGET_DIR}
fi
LIBOBLOG_RPM_NAME=$2
echo "dep_create.sh in ${PWD}, target dir: ${TARGET_DIR}, liboblog rpm name: ${LIBOBLOG_RPM_NAME}"
OS_ARCH="$(uname -p)" || exit 1
OS_RELEASE="0"
if [[ ! -f /etc/os-release ]]; then
echo "[ERROR] os release info not found" 1>&2 && exit 1
fi
source /etc/os-release || exit 1
PNAME=${PRETTY_NAME:-${NAME} ${VERSION}}
function compat_centos8() {
echo "[NOTICE] '$PNAME' is compatible with CentOS 8, use el8 dependencies list"
OS_RELEASE=8
}
function compat_centos7() {
echo "[NOTICE] '$PNAME' is compatible with CentOS 7, use el7 dependencies list"
OS_RELEASE=7
}
function not_supported() {
echo "[ERROR] '$PNAME' is not supported yet."
}
function version_ge() {
test "$(awk -v v1=$VERSION_ID -v v2=$1 'BEGIN{print(v1>=v2)?"1":"0"}' 2>/dev/null)" == "1"
}
function get_os_release() {
case "$ID" in
alinux)
version_ge "3.0" && compat_centos8 && return
version_ge "2.0" && compat_centos7 && return
;;
alios)
version_ge "8.0" && compat_centos8 && return
version_ge "7.2" && compat_centos7 && return
;;
anolis)
version_ge "8.0" && compat_centos8 && return
version_ge "7.0" && compat_centos7 && return
;;
ubuntu)
version_ge "16.04" && compat_centos7 && return
;;
centos)
version_ge "8.0" && OS_RELEASE=8 && return
version_ge "7.0" && OS_RELEASE=7 && return
;;
debian)
version_ge "9" && compat_centos7 && return
;;
fedora)
version_ge "33" && compat_centos7 && return
;;
opensuse-leap)
version_ge "15" && compat_centos7 && return
;;
#suse
sles)
version_ge "15" && compat_centos7 && return
;;
uos)
version_ge "20" && compat_centos7 && return
;;
esac
not_supported && return 1
}
get_os_release || exit 1
OS_TAG="el$OS_RELEASE.$OS_ARCH"
DEP_FILE="${PWD}/oblogproxy.${OS_TAG}.deps"
echo -e "check dependencies profile for ${OS_TAG}... \c"
if [[ ! -f "${DEP_FILE}" ]]; then
echo "NOT FOUND" 1>&2
exit 2
else
echo "FOUND"
fi
mkdir "${TARGET_DIR}/pkg" >/dev/null 2>&1
echo -e "check repository address in profile... \c"
REPO="$(grep -Po '(?<=kit_repo=).*' "${DEP_FILE}" 2>/dev/null)"
if [[ $? -eq 0 ]]; then
echo "$REPO"
else
echo "NOT FOUND" 1>&2
exit 3
fi
STABLE_REPO="$(grep -Po '(?<=stable_repo=).*' "${DEP_FILE}" 2>/dev/null)"
if [[ $? -eq 0 ]]; then
echo "$STABLE_REPO"
else
echo "NOT FOUND" 1>&2
exit 4
fi
echo "download dependencies..."
RPMS="$(grep '\.rpm' "${DEP_FILE}" | grep -Pv '^#')"
for pkg in $RPMS
do
if [[ -f "${TARGET_DIR}/pkg/${pkg}" ]]; then
echo -e "find package <${pkg}> in cache... \c"
else
echo -e "download package <${pkg}>... \c"
TEMP=$(mktemp -p "/" -u ".${pkg}.XXXX")
if [ "${LIBOBLOG_RPM_NAME}" == 'oceanbase-ce-devel' ] && [ ! -z `echo "${pkg}" | grep 'devdeps-mariadb-connector-c'` ]; then
echo "SKIP"
continue
fi
if [ ! -z `echo "${pkg}" | grep 'oceanbase-ce-devel'` ] || [ ! -z `echo "${pkg}" | grep 'oceanbase-ce-cdc'` ]; then
if [[ ! -z `echo "${pkg}" | grep ${LIBOBLOG_RPM_NAME}` ]]; then
echo ${TEMP}
wget "$STABLE_REPO/${pkg}" -q -O "${TARGET_DIR}/pkg/${TEMP}"
else
echo "SKIP"
continue
fi
else
wget "$REPO/${pkg}" -q -O "${TARGET_DIR}/pkg/${TEMP}"
fi
if [[ $? -eq 0 ]]; then
mv -f "${TARGET_DIR}/pkg/$TEMP" "${TARGET_DIR}/pkg/${pkg}"
echo "SUCCESS"
else
rm -rf "${TARGET_DIR}/pkg/$TEMP"
echo "FAILED" 1>&2
exit 5
fi
echo -e "unpack package <${pkg}>... \c"
cd ${TARGET_DIR} && rpm2cpio "${TARGET_DIR}/pkg/${pkg}" | cpio -di -u --quiet
fi
if [[ $? -eq 0 ]]; then
echo "SUCCESS"
else
echo "FAILED" 1>&2
exit 6
fi
done

13
deps/oblogproxy.el7.aarch64.deps vendored Normal file
View File

@ -0,0 +1,13 @@
[target]
os=7
arch=aarch64
kit_repo=http://mirrors.aliyun.com/oceanbase/development-kit/el/7/aarch64
stable_repo=https://mirrors.aliyun.com/oceanbase/community/stable/el/7/aarch64
[deps]
oceanbase-ce-cdc-4.2.0.0-100000052023080211.el7.aarch64.rpm
oceanbase-ce-devel-3.1.4-10000092022071511.el7.aarch64.rpm
[tools]
obdevtools-gcc9-9.3.0-52022092914.el7.aarch64.rpm
devdeps-openssl-static-1.0.1e-12022100422.el7.aarch64.rpm
devdeps-libaio-0.3.112-12022092915.el7.aarch64.rpm
devdeps-mariadb-connector-c-3.1.12-12022100422.el7.aarch64.rpm

15
deps/oblogproxy.el7.x86_64.deps vendored Normal file
View File

@ -0,0 +1,15 @@
[target]
os=7
arch=x86_64
kit_repo=http://mirrors.aliyun.com/oceanbase/development-kit/el/7/x86_64/
stable_repo=https://mirrors.aliyun.com/oceanbase/community/stable/el/7/x86_64/
[deps]
devdeps-openssl-static-1.0.1e-12022100422.el7.x86_64.rpm
devdeps-libaio-0.3.112-12022092915.el7.x86_64.rpm
devdeps-mariadb-connector-c-3.1.12-12022100422.el7.x86_64.rpm
oceanbase-ce-cdc-4.2.0.0-100000052023080211.el7.x86_64.rpm
oceanbase-ce-devel-3.1.4-10000092022071511.el7.x86_64.rpm
[tools]
obdevtools-gcc9-9.3.0-52022092914.el7.x86_64.rpm

15
deps/oblogproxy.el8.x86_64.deps vendored Normal file
View File

@ -0,0 +1,15 @@
[target]
os=8
arch=x86_64
kit_repo=http://mirrors.aliyun.com/oceanbase/development-kit/el/8/x86_64/
stable_repo=https://mirrors.aliyun.com/oceanbase/community/stable/el/8/x86_64/
[deps]
devdeps-openssl-static-1.0.1e-12022100422.el8.x86_64.rpm
devdeps-libaio-0.3.112-12022092915.el8.x86_64.rpm
devdeps-mariadb-connector-c-3.1.12-12022100422.el8.x86_64.rpm
oceanbase-ce-cdc-4.2.0.0-100000042023072615.el7.x86_64.rpm
oceanbase-ce-devel-3.1.4-10000092022071511.el8.x86_64.rpm
[tools]
obdevtools-gcc9-9.3.0-52022092914.el8.x86_64.rpm

64
docs/build.md Normal file
View File

@ -0,0 +1,64 @@
## How to Build
### Preparation
#### Install CMake
Install [CMake](https://cmake.org/download) 3.20 or later.
#### Install Dependencies
Fedora based (including CentOS, Fedora, OpenAnolis, RedHat, UOS, etc.)
```bash
yum install which git wget rpm rpm-build cpio gcc gcc-c++ make glibc-devel glibc-headers libstdc++-static binutils openssl-devel libaio-devel
```
Debian based (including Debian, Ubuntu, etc.)
```bash
apt-get install git wget rpm rpm2cpio cpio gcc make build-essential binutils
```
### Build
Get the source code and execute the following commands in the project directory.
```bash
mkdir buildenv && cd buildenv
cmake ..
make -j 6
```
Then there should be a binary output `logproxy` in the current directory. You can set the working directory by the following commands.
```bash
mkdir -p ./oblogproxy/bin ./oblogproxy/run
cp -r ../conf ./oblogproxy/
cp ../script/run.sh ./oblogproxy/
cp logproxy ./oblogproxy/bin/
```
### Build Options
There are some build options.
| Option | Default | Description |
|---------------|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|
| WITH_DEBUG | ON | Debug mode flag. |
| WITH_ASAN | OFF | Flag of whether to build with [AddressSanitizer](https://github.com/google/sanitizers). |
| WITH_TEST | OFF | Flag of whether to build test. |
| WITH_DEMO | OFF | Flag of whether to build demo. |
| WITH_GLOG | ON | Flag of whether to build with glog. |
| WITH_DEPS | ON | Flag of whether to automatically download precompiled dependencies. |
| USE_LIBOBLOG | OFF | Flag of whether to build with a customized precompiled libobcdc/liboblog. |
| USE_OBCDC_NS | ON | Flag of whether to build with libobcdc, use 'OFF' to build with former liboblog (before 3.1.3). |
| USE_CXX11_ABI | ON | Flag of whether to build with C++11 ABI. Note that if precompiled dependencies are used, they need to be consistent, otherwise the symbols will not be found |
For example, if you want to build with precompiled libobcdc, you can use the following commands.
```bash
mkdir buildenv && cd buildenv
CMAKE_INCLUDE_PATH=/path/to/libobcdc CMAKE_LIBRARY_PATH=/path/to/libobcdc cmake -DUSE_LIBOBLOG=ON ..
make -j 6
```

65
docs/manual.md Normal file
View File

@ -0,0 +1,65 @@
## 依赖说明
默认情况下，会自动下载并编译依赖库。有几个点这里说明下：
- **openssl**：当前使用的版本是：1.0.1e，1.0.*和1.1.*版本API少量不兼容，当前liboblog和logproxy都是是基于1.0.*实现的，需要版本一致。
- **liboblog**：如前文描述，您也可以自行获取或编译，并由环境变量指定路径，运行时需要指定LD_LIBRARY_PATH。
- **libaio**：liboblog依赖。
## 链路加密
### oblogclient与oblogproxy间TLS通信
修改`conf.json`中以下字段：
- `channel_type`: "tls"，开启与oblogclient通信的TLS。
- `tls_ca_cert_file`: CA证书文件路径（绝对路径）
- `tls_cert_file`: 服务器端签名证书路径（绝对路径）
- `tls_key_file`: 服务器端的私钥路径（绝对路径）
- `tls_verify_peer`: true，开启oblogclient验证（绝对路径）
对应的oblogclient也需要相应配置，见 [oblogclient链路加密](https://github.com/oceanbase/oblogclient) 。
### oblogproxy(liboblog)与ObServer间TLS通信
修改`conf.json`中以下字段：
- `liboblog_tls`: true，开启与ObServer通信的TLS。
- `liboblog_tls_cert_path`: ObServer相关证书文件路径（绝对路径），可以拷贝ObServer部署文件路径下的`wallet`文件夹，需要确保此路径包含：ca.pem，client-cert.pem，client-key.pem
## 配置
通常，您只需要关心前文描述过的参数。对于其他参数，在不完全了解参数用途的情况下，不建议修改。
| 字段 | 默认值 | 说明 |
| ---- |----------| ---------- |
| service_port | 2983 | 服务端口 |
| encode_threadpool_size | 8 | 编码线程池初始化大小 |
| encode_queue_size | 20000 | 编码线程队列长度 |
| max_packet_bytes | 67108864 | 最大数据包字节数 |
| record_queue_size | 512 | 数据发送队列长度 |
| read_timeout_us | 2000000 | 数据读取队列批次等待周期，单位微秒 |
| read_fail_interval_us | 1000000 | 数据读取队列重试等待周期，单位微秒 |
| read_wait_num | 20000 | 数据读取队列批次等待数量 |
| send_timeout_us | 2000000 | 发送数据包超时，单位微秒 |
| send_fail_interval_us | 1000000 | 发送数据包失败重试等待周期，单位微秒 |
| command_timeout_s | 10 | 命令执行超时，单位微妙 |
| log_quota_size_mb | 5120 | 日志文件总大小阈值，单位MB |
| log_quota_day | 30 | 日志文件存储时间阈值，单位天 |
| log_gc_interval_s | 43200 | 日志文件清理周期，单位秒 |
| oblogreader_path_retain_hour | 168 | oblogreader子进程目录保留时间，单位小时 |
| oblogreader_lease_s | 300 | oblogreader子进程启动探测时间，单位秒 |
| oblogreader_path | ./run | oblogreader子进程上下文目录根路径 |
| allow_all_tenant | true | 是否允许订阅所有租户 |
| auth_user | true | 是否鉴权连接用户 |
| auth_use_rs | false | 是否通过root server鉴权用户 |
| auth_allow_sys_user | true | 是否允许订阅系统租户 |
| ob_sys_username | "" | 【必须自行配置】系统租户用户名密文，用来订阅增量 |
| ob_sys_password | "" | 【必须自行配置】系统租户密码密文，用来订阅增量 |
| counter_interval_s | 2 | 计数器周期，单位秒 |
| debug | false | 打印debug信息 |
| verbose | false | 打印更多debug信息 |
| verbose_packet | false | 打印数据包信息 |
| readonly | false | 只读模式 |
| channel_type | plain | 链路类型 |
| tls_ca_cert_file | "" | CA证书文件路径（绝对路径） |
| tls_cert_file | "" | 服务器端签名证书路径（绝对路径） |
| tls_key_file | "" | 服务器端的私钥路径（绝对路径） |
| tls_verify_peer | true | 开启oblogclient验证 |
| liboblog_tls | false | 开启与ObServer通信的TLS |
| liboblog_tls_cert_path | "" | ObServer相关证书文件路径（绝对路径）|

9
git-hooks/install-git-hooks.sh Executable file
View File

@ -0,0 +1,9 @@
#!/bin/bash
set -e
ROOT=$(git rev-parse --show-toplevel)
ln -sf $ROOT/git-hooks/pre-commit $ROOT/.git/hooks/pre-commit
echo "installed git-hooks/pre-commit to .git/hooks/pre-commit"

8
git-hooks/pre-commit Executable file
View File

@ -0,0 +1,8 @@
#!/bin/bash
set -e
ROOT=$(git rev-parse --show-toplevel)
CLANG_FORMAT_HOOK_SCRIPT=$ROOT/git-hooks/pre-commit-clang-format
$CLANG_FORMAT_HOOK_SCRIPT

67
git-hooks/pre-commit-clang-format Executable file
View File

@ -0,0 +1,67 @@
#!/bin/bash
CLANG_FORMAT=`which clang-format`
SOURCE_FILE_EXTS=(.h .hh .hpp .hxx .c .cc .cpp .cxx)
check_clang_format() {
if [ ! -x "$CLANG_FORMAT" ] ; then
echo "ERROR: clang-format executable not found."
exit 1
fi
}
# check whether the given file matches any of the extensions: case insensitive
matches_extension() {
local file_name=$(basename "$1")
local file_ext=".${file_name##*.}"
local lowercase_file_ext=`echo $file_ext | awk '{print tolower($0)}'`
local source_file_ext
for source_file_ext in "${SOURCE_FILE_EXTS[@]}"
do
local lowercase_source_file_ext=`echo $source_file_ext | awk '{print tolower($0)}'`
[[ "$lowercase_file_ext" = "$lowercase_source_file_ext" ]] && return 0
done
return 1
}
_FORMATTED_FILES_CNT=0
format_file() {
local source_file=$1
local formatted_file="${source_file}.formatted"
$CLANG_FORMAT -style=file $source_file > $formatted_file
cmp -s $source_file $formatted_file
if [ $? -ne 0 ]
then
mv $formatted_file $source_file && echo "formatted file $source_file"
let _FORMATTED_FILES_CNT++
else
rm -f $formatted_file
fi
}
_ROOT=$(git rev-parse --show-toplevel)
format_staged_source_files() {
for file in $(git diff --staged --name-only)
do
file=$_ROOT/$file
matches_extension $file && format_file $file
done
}
# return 1 if there was any file actually formatted, which will break the `git commit` process.
verify_format() {
if [ ${_FORMATTED_FILES_CNT} -gt 0 ]
then
echo "${_FORMATTED_FILES_CNT} files has been formatted."
exit 1
fi
}
main() {
check_clang_format
format_staged_source_files
verify_format
}
main

30
package.sh Normal file
View File

@ -0,0 +1,30 @@
#!/usr/bin/env
ce=$1
version=$2
if [ -z "${ce}" ] || [ -z "${version}" ]; then
echo "Invalid input arguments"
exit -1
fi
file_path=$(dirname $0 | xargs readlink -f)
function compile_ce() {
liboblog_3x_flag=$1
name=$2
echo "building and packaging ${name}"
cd ${file_path} && \
mkdir -p packenv && cd packenv && \
ls | grep -v *.tar.gz | xargs rm -rf && \
cmake -DOBLOGPROXY_INSTALL_PREFIX=`pwd`/oblogproxy -DUSE_LIBOBLOG_3=${liboblog_3x_flag} .. && \
make -j $(grep -c ^processor /proc/cpuinfo) install oblogproxy && \
tar -zcf ${name}.tar.gz oblogproxy
}
tm=$(date +%Y%m%d%H%M%S)
compile_ce ON oblogproxy-ce-for-3x-${version}-${tm}
compile_ce OFF oblogproxy-ce-for-4x-${version}-${tm}

6
proto/gen.sh Normal file
View File

@ -0,0 +1,6 @@
set -x
cd $(dirname $0)/..
dir=$(pwd)
protoc -I. --java_out=${dir}/common/src/main/java proto/logproxy.proto
protoc -I. --cpp_out=${dir}/src proto/logproxy.proto

30
proto/legacy.proto Normal file
View File

@ -0,0 +1,30 @@
syntax = "proto3";
package oceanbase.logproxy.legacy;
option java_package = "com.oceanbase.clogproxy.common.packet.protocol";
option java_outer_classname = "LogProxyProto";
message PbPacket {
int32 type = 1; // HeaderType
int32 compress_type = 2; // CompressType
// resevered for other options
bytes payload = 100;
}
message ClientHandShake {
int32 log_type = 1; // LogType
string client_ip = 2;
string client_id = 3;
string client_version = 4;
bool enable_monitor = 5;
string configuration = 6;
}
message RuntimeStatus {
string ip = 1;
int32 port = 2;
int32 stream_count = 3;
int32 worker_count = 4;
}

44
proto/logproxy.proto Normal file
View File

@ -0,0 +1,44 @@
syntax = "proto3";
package oceanbase.logproxy;
option java_package = "com.oceanbase.clogproxy.common.packet.protocol";
option java_outer_classname = "LogProxyProto";
message ErrorResponse {
int32 code = 1;
string message = 2;
}
message ClientHandshakeRequest {
int32 log_type = 1; // LogType
string ip = 2;
string id = 3;
string version = 4;
bool enable_monitor = 5;
string configuration = 6;
}
message ClientHandshakeResponse {
int32 code = 1;
string ip = 2;
string version = 3;
}
message RuntimeStatus {
string ip = 1;
int32 port = 2;
int32 stream_count = 3;
int32 worker_count = 4;
}
message RecordData {
int32 compress_type = 1; // CompressType
int32 raw_len = 2;
int32 compressed_len = 3;
int32 count = 4;
// resevered for other options
bytes records = 100;
}

51
rpm/oblogproxy-build.sh Normal file
View File

@ -0,0 +1,51 @@
#!/bin/bash
CUR_DIR=$(dirname $(readlink -f "$0"))
PROJECT_DIR=${1:-${CUR_DIR}/../}
PROJECT_NAME=$2
VERSION=$3
RELEASE=$4
CPU_CORES=`grep -c ^processor /proc/cpuinfo`
echo "[BUILD] args: CURDIR=${CUR_DIR} PROJECT_NAME=${PROJECT_NAME} VERSION=${VERSION} RELEASE=${RELEASE}"
# inject env variables
export PROJECT_NAME=${PROJECT_NAME}
export VERSION=${VERSION}
export RELEASE=${RELEASE}
# prepare building env
cd $CUR_DIR
DEP_DIR=$CUR_DIR/deps
mkdir -p $DEP_DIR
OS_ARCH=$(uname -m)
OS_RELEASE=$(grep -Po '(?<=release )\d' /etc/redhat-release)
OS_TAG=${OS_ARCH}/${OS_RELEASE}
CMAKE_COMMAND=cmake
case $OS_TAG in
x86_64/7)
wget https://mirrors.aliyun.com/oceanbase/development-kit/el/7/x86_64/obdevtools-cmake-3.20.2-3.el7.x86_64.rpm -P $DEP_DIR
rpm2cpio ${DEP_DIR}/obdevtools-cmake-3.20.2-3.el7.x86_64.rpm | cpio -idvm
CMAKE_COMMAND=${CUR_DIR}/usr/local/oceanbase/devtools/bin/cmake
;;
x86_64/8)
wget https://mirrors.aliyun.com/oceanbase/development-kit/el/8/x86_64/obdevtools-cmake-3.20.2-3.el8.x86_64.rpm -P $DEP_DIR
rpm2cpio ${DEP_DIR}/obdevtools-cmake-3.20.2-3.el8.x86_64.rpm | cpio -idvm
CMAKE_COMMAND=${CUR_DIR}/usr/local/oceanbase/devtools/bin/cmake
;;
**)
echo "Unsupported os arch, please prepare the building environment in advance."
;;
esac
# build rpm
cd $PROJECT_DIR
rm -rf build_rpm
mkdir build_rpm
cd build_rpm
${CMAKE_COMMAND} .. -DOBLOGPROXY_RELEASEID=$RELEASE -DUSE_OBCDC_NS=ON -DOBLOGPROXY_PACKAGE_NAME=$PROJECT_NAME -DOBLOGPROXY_PACKAGE_VERSION=$VERSION
make -j${CPU_CORES} rpm
# archiving artifacts
cd $CUR_DIR
find ${PROJECT_DIR}/build_rpm -name "*.rpm" -maxdepth 1 -exec mv {} . 2>/dev/null \;

138
script/run.sh Executable file
View File

@ -0,0 +1,138 @@
#!/usr/bin/env bash
cd $(dirname $0)
DEPLOY_PATH=$(pwd)
echo "DEPLOY_PATH : "${DEPLOY_PATH}
LIB_PATH=${DEPLOY_PATH}/liboblog
BIN='logproxy'
GPID=0
function is_running() {
for pid in $(ps ux | grep ${BIN} | grep -v 'bash ' | grep -v grep | awk '{print $2}'); do
_path=$(readlink -f /proc/${pid}/cwd)
if [[ ${?} -eq 0 ]] && [[ ${_path} == ${DEPLOY_PATH} ]]; then
GPID=${pid}
echo "is_running : (${GPID})${DEPLOY_PATH} ${BIN} is running!"
return 1
fi
done
return 0
}
function kill_proc_9() {
force=$1
if [[ ! -d ${DEPLOY_PATH} ]]; then
echo "kill_proc : ${DEPLOY_PATH} invalid!"
return 0
fi
retry=0
is_running
status=$?
while [[ ${status} -eq 1 ]]; do
if [ ! -z ${force} ]; then
kill -9 ${GPID}
echo "kill_proc force : (${GPID})${DEPLOY_PATH} succ!"
else
kill ${GPID}
echo "kill_proc : (${GPID})${DEPLOY_PATH} succ!"
fi
sleep 1
retry=$(expr ${retry} + 1)
if [ ${retry} -gt 15 ]; then
force=1
fi
is_running
status=$?
done
return 0
}
start() {
stop
if [ ! -d "./run" ]; then
mkdir ./run
fi
log_path="./log"
if [ ! -d ${log_path} ]; then
mkdir ${log_path}
fi
log_path=$(readlink -f ${log_path})
if [ ! -z "${LIB_PATH}" ]; then
export LD_LIBRARY_PATH=${LIB_PATH}:${LD_LIBRARY_PATH}
fi
chmod u+x ./bin/${BIN} && ./bin/${BIN} -f ./conf/conf.json &>${log_path}/out.log &
if [ $? -ne 0 ]; then
exit -1
fi
is_running
}
stop() {
kill_proc_9
}
do_config_sys() {
username=$1
password=$2
if [[ -z "${username}" ]] || [[ -z "${password}" ]]; then
echo "No input sys username or password"
exit -1
fi
if [ ! -z "${LIB_PATH}" ]; then
export LD_LIBRARY_PATH=${LIB_PATH}:${LD_LIBRARY_PATH}
fi
username_x=`./bin/${BIN} -x ${username}`
password_x=`./bin/${BIN} -x ${password}`
cp ./conf/conf.json ./conf/conf.json.new
sed -r -i 's/"ob_sys_username"[ ]*:[ ]*"[0-9a-zA-Z]*/"ob_sys_username": "'${username_x}'/' ./conf/conf.json.new
sed -r -i 's/"ob_sys_password"[ ]*:[ ]*"[0-9a-zA-Z]*/"ob_sys_password": "'${password_x}'/' ./conf/conf.json.new
diff ./conf/conf.json ./conf/conf.json.new
echo ""
read -r -p "!!DANGER!! About to update logproxy conf/conf.json, Please confirm? [Y/n] " response
if [ "${response}" != "y" ] && [ "${response}" != "Y" ]; then
echo "Cancel!"
rm -rf ./conf/conf.json.new
exit 0
fi
cp ./conf/conf.json ./conf/conf.json.bak
mv ./conf/conf.json.new ./conf/conf.json
}
case C"$1" in
Cstop)
stop
echo "${BIN} stopped!"
;;
Cstart)
start
echo "${BIN} started!"
;;
Cdebug)
debug
echo "${BIN} started!"
;;
Cstatus)
is_running
status=$?
echo "status : ${status}"
exit ${status}
;;
Cconfig_sys)
do_config_sys $2 $3
;;
C*)
echo "Usage: $0 {start|stop|status}"
;;
esac

355
src/arranger/arranger.cpp Normal file
View File

@ -0,0 +1,355 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <sys/wait.h>
#include <arpa/inet.h>
#include <signal.h>
#include <math.h>
#include <memory>
#include "common/log.h"
#include "arranger/source_invoke.h"
#include "arranger/arranger.h"
#include "obaccess/ob_access.h"
#include "obaccess/clog_meta_routine.h"
#include "metric/status_thread.h"
#include "metric/sys_metric.h"
namespace oceanbase {
namespace logproxy {
SysMetric g_metric;
static Config& _s_conf = Config::instance();
int Arranger::init()
{
if (!localhostip(_localhost, _localip) || _localhost.empty() || _localip.empty()) {
OMS_ERROR << "Failed to fetch localhost name or localip";
return OMS_FAILED;
}
int ret = ChannelFactory::instance().init(_s_conf);
if (ret != OMS_OK) {
OMS_ERROR << "Failed to init channel factory";
return OMS_FAILED;
}
ret = _accepter.init();
if (ret == OMS_FAILED) {
return ret;
}
_accepter.set_read_callback([this](const Peer& peer, const Message& msg) { return on_handshake(peer, msg); });
_accepter.set_routine_callback([this] { gc_pid_routine(); });
_accepter.set_close_callback([this](const Peer& peer) { on_close(peer); });
return OMS_OK;
}
int Arranger::run_foreground()
{
StatusThread status_thread(g_metric);
status_thread.register_gauge("NREADER", [this] { return _client_peers.size(); });
status_thread.register_gauge("NCHANNEL", [this] { return _accepter.channel_count(); });
if (_s_conf.metric_enable.val()) {
status_thread.start();
status_thread.detach();
}
int ret = _accepter.listen(_s_conf.service_port.val());
if (ret != OMS_OK) {
return ret;
}
return _accepter.start();
}
EventResult Arranger::on_handshake(const Peer& peer, const Message& msg)
{
OMS_INFO << "Arranger on_msg fired: " << peer.to_string();
if (msg.type() != MessageType::HANDSHAKE_REQUEST_CLIENT) {
OMS_WARN << "Unknown message type: " << (int)msg.type();
return EventResult::ER_CLOSE_CHANNEL;
}
auto& handshake = (ClientHandshakeRequestMessage&)msg;
OMS_INFO << "Handshake request from peer: " << peer.to_string() << ", msg: " << handshake.to_string();
ClientMeta client = ClientMeta::from_handshake(peer, handshake);
client.packet_version = msg.version();
std::string errmsg;
OblogConfig oblog_config(client.configuration);
if (resolve(oblog_config, errmsg) != OMS_OK) {
response_error(peer, msg.version(), ErrorCode::NO_AUTH, errmsg);
return EventResult::ER_CLOSE_CHANNEL;
}
OMS_INFO << "ObConfig from peer: " << peer.to_string() << " after resolve: " << oblog_config.debug_str();
if (auth(oblog_config, errmsg) != OMS_OK) {
response_error(peer, msg.version(), ErrorCode::NO_AUTH, errmsg);
return EventResult::ER_CLOSE_CHANNEL;
}
// resolve sys user
if (!oblog_config.sys_user.empty()) {
oblog_config.user.set(oblog_config.sys_user.val());
} else {
oblog_config.user.set(Config::instance().ob_sys_username.val());
}
if (check_quota() != OMS_OK) {
// close connect directly to make client reconnect
return EventResult::ER_CLOSE_CHANNEL;
}
ClogMetaRoutine clog_meta;
if (clog_meta.init(oblog_config) != OMS_OK) {
response_error(peer, msg.version(), ErrorCode::NO_AUTH, "Failed to init clog check");
return EventResult::ER_CLOSE_CHANNEL;
}
if (!clog_meta.check(oblog_config, errmsg)) {
OMS_ERROR << errmsg;
response_error(peer, msg.version(), ErrorCode::NO_AUTH, errmsg);
return EventResult::ER_CLOSE_CHANNEL;
}
ClientHandshakeResponseMessage resp(0, _localip, __OMS_VERSION__);
resp.set_version(msg.version());
int ret = _accepter.send_message(peer, resp, true);
if (ret != OMS_OK) {
OMS_WARN << "Failed to send handshake response message. peer: " << peer.to_string();
return EventResult::ER_CLOSE_CHANNEL;
}
client.peer = peer;
ret = create(client, oblog_config);
if (ret != OMS_OK) {
response_error(peer, msg.version(), E_INNER, "Failed to create oblogreader");
return EventResult::ER_CLOSE_CHANNEL;
}
return EventResult::ER_SUCCESS;
}
int Arranger::resolve(OblogConfig& hs_config, std::string& errmsg)
{
hex2bin(hs_config.password.val().c_str(), hs_config.password.val().size(), hs_config.password_sha1);
if (!hs_config.sys_password.empty()) {
hs_config.password.set(hs_config.sys_password.val());
MysqlProtocol::do_sha_password(hs_config.sys_password.val(), hs_config.sys_password_sha1);
} else {
hs_config.password.set(Config::instance().ob_sys_password.val());
MysqlProtocol::do_sha_password(Config::instance().ob_sys_password.val(), hs_config.sys_password_sha1);
}
if (!hs_config.root_servers.empty()) {
// do nothing
} else if (!Config::instance().builtin_cluster_url_prefix.val().empty()) {
// resolve builtin cluster url
if (hs_config.cluster_url.empty()) {
if (hs_config.cluster_id.empty()) {
errmsg = "Refuse connection caused by no cluster id or cluster url";
OMS_ERROR << errmsg;
return OMS_FAILED;
}
hs_config.cluster_url.set(Config::instance().builtin_cluster_url_prefix.val() + hs_config.cluster_id.val());
hs_config.cluster_id.set("");
}
} else if (hs_config.cluster_url.empty()) {
errmsg = "Refuse connection caused by no cluster url";
OMS_ERROR << errmsg;
return OMS_FAILED;
}
return OMS_OK;
}
int Arranger::auth(const OblogConfig& oblog_config, std::string& errmsg)
{
if (_s_conf.auth_user.val()) {
ObAccess ob_access;
int ret = ob_access.init(oblog_config, oblog_config.password_sha1, oblog_config.sys_password_sha1);
if (ret != OMS_OK) {
errmsg = "Failed to parse configuration";
return ret;
}
ret = ob_access.auth();
if (ret != OMS_OK) {
errmsg = "Failed to auth";
return ret;
}
}
return OMS_OK;
}
int Arranger::check_quota()
{
if (!_s_conf.check_quota_enable.val()) {
return OMS_OK;
}
if (_client_peers.size() >= _s_conf.oblogreader_max_count.val()) {
OMS_ERROR << "Exceed max oblogreader count, current: " << _client_peers.size();
return OMS_FAILED;
}
int max_cpu_ratio = _s_conf.max_cpu_ratio.val();
int current_cpu_ratio = ::floor(g_metric.cpu_status.cpu_used_ratio);
if (max_cpu_ratio != 0 && current_cpu_ratio >= max_cpu_ratio) {
OMS_ERROR << "Exceed max cpu ratio: " << max_cpu_ratio << ", current: " << current_cpu_ratio;
return OMS_FAILED;
}
uint64_t max_mem_quota_mb = _s_conf.max_mem_quota_mb.val();
uint64_t current_mem_mb = g_metric.memory_status.mem_used_size_mb;
if (max_mem_quota_mb != 0 && current_mem_mb >= max_mem_quota_mb) {
OMS_ERROR << "Exceed max mem quota in MB: " << max_mem_quota_mb << ", current: " << current_mem_mb;
return OMS_FAILED;
}
return OMS_OK;
}
int Arranger::create(ClientMeta& client, const OblogConfig& oblog_config)
{
OMS_INFO << "Client connecting: " << client.to_string();
const std::string& client_id = client.id;
const auto& fd_entry = _client_peers.find(client_id);
if (fd_entry != _client_peers.end()) {
OMS_WARN << "Duplication exist clientId: " << client_id << ", close last one: " << fd_entry->second.peer.id()
<< " with pid: " << fd_entry->second.pid;
return OMS_FAILED;
// close_client_force(fd_entry->second, "Duplication exist client_id");
}
int ret = SourceInvoke::invoke(_accepter, client, oblog_config);
if (ret <= 0) {
OMS_ERROR << "Failed to start source of client:" << client.to_string();
return OMS_FAILED;
}
_accepter.del(client.peer); // remove fd event for parent;
OMS_INFO << "Remove peer: " << client.peer.to_string()
<< " after source invoked, current channel count:" << _accepter.channel_count();
client.pid = ret;
_client_peers.emplace(client_id, client);
OMS_INFO << "Client connected: " << client_id << " with peer: " << client.peer.to_string();
return OMS_OK;
}
void Arranger::response_error(const Peer& peer, MessageVersion version, ErrorCode code, const std::string& errmsg)
{
ErrorMessage error(code, errmsg);
error.set_version(version);
int ret = _accepter.send_message(peer, error, true);
if (ret != OMS_OK) {
OMS_WARN << "Failed to send error response message to peer:" << peer.to_string() << " for message:" << errmsg;
}
}
void Arranger::on_close(const Peer& peer)
{
for (auto iter = _client_peers.begin(); iter != _client_peers.end(); ++iter) {
if (iter->second.peer.id() == peer.id()) {
OMS_WARN << "On close peer fd: " << peer.fd << " with client: " << iter->second.id;
// As fd was sent to child process and close immediately,
// we do del again here for checking if fd was leaked caused by potential bugs
_accepter.del(iter->second.peer);
int fd = iter->second.peer.fd;
OMS_WARN << "Try to shutdown fd: " << fd;
shutdown(fd, SHUT_RDWR);
OMS_WARN << "Shutdown fd: " << fd;
_client_peers.erase(iter);
break;
}
}
}
int Arranger::close_client_force(const ClientMeta& client, const std::string& msg)
{
ErrorMessage err(OMS_FAILED, msg);
err.set_version(client.packet_version);
_accepter.trigger_del(client.peer, err);
int fd = client.peer.fd;
OMS_WARN << "Try to shutdown fd: " << fd;
shutdown(fd, SHUT_RDWR);
OMS_WARN << "Shutdown fd: " << fd;
auto entry = _client_peers.find(client.id);
if (entry != _client_peers.end()) {
int pid = entry->second.pid;
if (pid != 0) {
// make sure last oblogreader was exit
// trigger child_waiter later
// then GC by gc_pid_routine and call close_by_pid
// as we called close() in trigger_del already, time window wthin that
// there would be new client conneted again
// causing current client with same fd closed by later GC,
// we avoid this phantom scenario by check peer id
::kill(pid, SIGKILL);
close_by_pid(pid, client);
}
_client_peers.erase(entry);
}
return OMS_OK;
}
void Arranger::gc_pid_routine()
{
for (auto iter = _client_peers.begin(); iter != _client_peers.end();) {
int pid = iter->second.pid;
// detect if oblogreader still alive
if (kill(pid, 0) != 0) {
close_by_pid(pid, iter->second);
iter = _client_peers.erase(iter);
} else {
++iter;
}
}
}
void Arranger::close_by_pid(int pid, const ClientMeta& client)
{
OMS_WARN << "Exited oblogreader of pid: " << pid << " with clientId: " << client.id
<< " of peer:" << client.peer.to_string();
int fd = client.peer.fd;
struct sockaddr_in peer_addr;
socklen_t len;
int ret = getpeername(fd, (struct sockaddr*)&peer_addr, &len);
if (ret == 0 && peer_addr.sin_addr.s_addr != 0) {
Peer p(peer_addr.sin_addr.s_addr, ntohs(peer_addr.sin_port), fd);
OMS_INFO << "fetched peer: " << p.to_string();
if (p.id() != client.peer.id()) {
// fd number was assigned to another(new connected) client session
return;
}
} else {
OMS_WARN << "Failed to fetch peer info of fd:" << fd << ", errno:" << errno << ", error:" << strerror(errno);
}
// As fd was sent to child process and close immediately,
// we do del again here for checking if fd was leaked caused by potential bugs
_accepter.del(client.peer);
OMS_WARN << "Try to shutdown fd: " << fd;
shutdown(fd, SHUT_RDWR);
OMS_WARN << "Shutdown fd: " << fd;
}
} // namespace logproxy
} // namespace oceanbase

71
src/arranger/arranger.h Normal file
View File

@ -0,0 +1,71 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <unordered_map>
#include <mutex>
#include "common/common.h"
#include "obaccess/oblog_config.h"
#include "arranger/source_meta.h"
#include "arranger/client_meta.h"
namespace oceanbase {
namespace logproxy {
class Arranger {
OMS_SINGLETON(Arranger);
OMS_AVOID_COPY(Arranger);
public:
int init();
int run_foreground();
private:
void on_close(const Peer&);
EventResult on_handshake(const Peer&, const Message&);
int resolve(OblogConfig&, std::string& errmsg);
int auth(const OblogConfig&, std::string& errmsg);
int check_quota();
int create(ClientMeta&, const OblogConfig&);
void response_error(const Peer&, MessageVersion version, ErrorCode code, const std::string&);
int close_client_force(const ClientMeta& client, const std::string& msg = "");
void close_by_pid(int pid, const ClientMeta& client);
void gc_pid_routine();
private:
/**
* <ClientId, sink_peer>
*/
std::unordered_map<std::string, ClientMeta> _client_peers;
std::string _localhost;
std::string _localip;
Comm _accepter;
};
class SysMetric;
extern SysMetric g_metric;
} // namespace logproxy
} // namespace oceanbase

39
src/arranger/client_meta.cpp Normal file
View File

@ -0,0 +1,39 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "arranger/client_meta.h"
namespace oceanbase {
namespace logproxy {
ClientMeta ClientMeta::from_handshake(const Peer& peer, ClientHandshakeRequestMessage& handshake)
{
ClientMeta meta;
meta.type = (LogType)handshake.log_type;
meta.id = handshake.id;
meta.ip = handshake.ip;
meta.version = handshake.version;
meta.configuration = handshake.configuration;
meta.peer = peer;
meta.register_time = time(nullptr);
return meta;
}
LogStream& operator<<(LogStream& ss, MessageVersion version)
{
ss << (uint16_t)version;
return ss;
}
} // namespace logproxy
} // namespace oceanbase

53
src/arranger/client_meta.h Normal file
View File

@ -0,0 +1,53 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <string>
#include <ostream>
#include "common/model.h"
#include "common/config.h"
#include "communication/comm.h"
#include "codec/message.h"
namespace oceanbase {
namespace logproxy {
struct ClientMeta : public Model {
OMS_MF_ENABLE_COPY(ClientMeta);
public:
ClientMeta() = default;
OMS_MF(LogType, type);
OMS_MF(std::string, id);
OMS_MF(std::string, ip);
OMS_MF(std::string, version);
OMS_MF(std::string, configuration);
OMS_MF_DFT(int, pid, 0);
OMS_MF(Peer, peer);
OMS_MF(time_t, register_time);
OMS_MF_DFT(bool, enable_monitor, false);
OMS_MF_DFT(MessageVersion, packet_version, MessageVersion::V2);
public:
static ClientMeta from_handshake(const Peer&, ClientHandshakeRequestMessage&);
};
LogStream& operator<<(LogStream& ss, MessageVersion version);
} // namespace logproxy
} // namespace oceanbase

95
src/arranger/source_invoke.cpp Normal file
View File

@ -0,0 +1,95 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <unistd.h>
#include <signal.h>
#ifdef linux
#include <sys/prctl.h>
#endif
#include "common/log.h"
#include "common/config.h"
#include "common/fs_util.h"
#include "oblogreader/oblogreader.h"
#include "arranger/source_invoke.h"
#include "arranger/arranger.h"
namespace oceanbase {
namespace logproxy {
static int start_oblogreader(Comm& comm, const ClientMeta& client, const OblogConfig& config)
{
int pid = fork();
if (pid == -1) {
OMS_ERROR << "failed to fork: " << strerror(errno);
return OMS_FAILED;
}
if (pid == 0) { // children
char* process_name_addr = (char*)Config::instance().process_name_address.val();
const char* child_process_name = "oblogreader";
#ifdef linux
::prctl(PR_SET_NAME, (unsigned long)child_process_name);
#endif
strncpy(process_name_addr, child_process_name, strlen(process_name_addr));
// change process context dir path
std::string process_path = Config::instance().oblogreader_path.val() + std::string("/") + client.id;
FsUtil::mkdir(process_path);
FsUtil::mkdir(process_path + "/log");
::chdir(process_path.c_str());
// reload log
init_log(child_process_name, true);
OMS_INFO << "!!! Started oblogreader process(" << getpid() << ") with peer: " << client.peer.to_string()
<< ", client meta:" << client.to_string();
comm.stop(client.peer.fd);
// we create new thread for fork() acting as children process's main thread
// child never return as exit internal if error
ObLogReader reader;
int ret = reader.init(client.id, client.packet_version, client, config);
if (ret == OMS_OK) {
reader.start();
reader.join();
}
// !!!IMPORTANT!!! we don't quit current thread which work as child process's main thread
// we IGNORE other context inheriting from parent process
OMS_WARN << "!!! Exiting oblogreader process(" << getpid() << ") with peer: " << client.peer.to_string()
<< ", client meta:" << client.to_string();
::exit(-1);
}
OMS_INFO << "+++ Created oblogreader with pid: " << pid;
return pid;
}
/**
* @return pid of childern process or -1 failurs, childern process never return
*/
int SourceInvoke::invoke(Comm& comm, const ClientMeta& client, const OblogConfig& config)
{
switch (client.type) {
case OCEANBASE:
return start_oblogreader(comm, client, config);
default:
OMS_ERROR << "Unsupported invoke logtype: " << client.type;
return OMS_FAILED;
}
}
} // namespace logproxy
} // namespace oceanbase

30
src/arranger/source_invoke.h Normal file
View File

@ -0,0 +1,30 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include "common/config.h"
#include "common/thread.h"
#include "obaccess//oblog_config.h"
#include "arranger/client_meta.h"
namespace oceanbase {
namespace logproxy {
class SourceInvoke {
public:
static int invoke(Comm&, const ClientMeta&, const OblogConfig&);
};
} // namespace logproxy
} // namespace oceanbase

54
src/arranger/source_meta.h Normal file
View File

@ -0,0 +1,54 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <string>
#include "arranger/client_meta.h"
namespace oceanbase {
namespace logproxy {
struct SourceMeta {
public:
const LogType type;
const int pid;
const std::string client_id;
/**
* last checkd alive time
*/
time_t _last_time;
public:
SourceMeta(LogType t, int p, const std::string& cid) : type(t), pid(p), client_id(cid)
{
id_str = std::to_string(pid);
}
std::string to_string() const
{
return "";
}
const std::string& id() const
{
return id_str;
}
private:
std::string id_str;
};
} // namespace logproxy
} // namespace oceanbase

60
src/codec/codec_endian.h Normal file
View File

@ -0,0 +1,60 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <stdint.h>
namespace oceanbase {
namespace logproxy {
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define le_to_cpu(integer) (integer)
#define cpu_to_le(integer) (integer)
inline uint16_t be_to_cpu(uint16_t integer)
{
return __builtin_bswap16(integer);
}
inline uint32_t be_to_cpu(uint32_t integer)
{
return __builtin_bswap32(integer);
}
inline uint64_t be_to_cpu(uint64_t integer)
{
return __builtin_bswap64(integer);
}
inline uint16_t cpu_to_be(uint16_t integer)
{
return __builtin_bswap16(integer);
}
inline uint32_t cpu_to_be(uint32_t integer)
{
return __builtin_bswap32(integer);
}
inline uint64_t cpu_to_be(uint64_t integer)
{
return __builtin_bswap64(integer);
}
#else
#endif
} // namespace logproxy
} // namespace oceanbase

62
src/codec/decoder.h Normal file
View File

@ -0,0 +1,62 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include "codec/message.h"
#include "codec/msg_buf.h"
namespace oceanbase {
namespace logproxy {
class Channel;
class MessageDecoder {
public:
virtual PacketError decode(Channel& ch, MessageVersion version, Message*& message) = 0;
protected:
size_t _header_len;
};
class LegacyDecoder : public MessageDecoder {
OMS_AVOID_COPY(LegacyDecoder);
OMS_SINGLETON(LegacyDecoder);
public:
PacketError decode(Channel& ch, MessageVersion version, Message*& message) override;
private:
static int decode_handshake_request(Channel& ch, Message*& message);
};
class ProtobufDecoder : public MessageDecoder {
OMS_AVOID_COPY(ProtobufDecoder);
OMS_SINGLETON(ProtobufDecoder);
public:
PacketError decode(Channel& ch, MessageVersion version, Message*& message) override;
private:
int decode_payload(MessageType type, const MsgBuf& buffer, Message*& msg);
static int decode_handshake_request(MsgBufReader& buffer_reader, Message*& msg);
static int decode_handshake_response(MsgBufReader& buffer_reader, Message*& msg);
static int decode_runtime_status(MsgBufReader& buffer_reader, Message*& msg);
static int decode_data_client(MsgBufReader& buffer_reader, Message*& msg);
};
} // namespace logproxy
} // namespace oceanbase

76
src/codec/encoder.h Normal file
View File

@ -0,0 +1,76 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include "codec/message.h"
#include "codec/msg_buf.h"
namespace google {
namespace protobuf {
class Message;
} // namespace protobuf
} // namespace google
namespace oceanbase {
namespace logproxy {
class MessageEncoder {
public:
virtual ~MessageEncoder() = default;
virtual int encode(const Message& msg, MsgBuf& buffer, size_t& raw_len) = 0;
};
class LegacyEncoder : public MessageEncoder {
OMS_AVOID_COPY(LegacyEncoder);
public:
static LegacyEncoder& instance()
{
static LegacyEncoder singleton;
return singleton;
}
private:
LegacyEncoder();
public:
int encode(const Message& msg, MsgBuf& buffer, size_t& raw_len) override;
private:
std::unordered_map<int8_t, std::function<int(const Message&, MsgBuf&, size_t&)>> _funcs;
};
class ProtobufEncoder : public MessageEncoder {
OMS_AVOID_COPY(ProtobufEncoder);
OMS_SINGLETON(ProtobufEncoder);
public:
int encode(const Message& msg, MsgBuf& buffer, size_t& raw_len) override;
private:
static int encode_message(const google::protobuf::Message& pb_msg, MessageType type, MsgBuf& buffer, bool magic);
static int encode_error_response(const Message& msg, MsgBuf& buffer);
static int encode_client_handshake_request(const Message& msg, MsgBuf& buffer);
static int encode_client_handshake_response(const Message& msg, MsgBuf& buffer);
static int encode_runtime_status(const Message& msg, MsgBuf& buffer, size_t& raw_len);
static int encode_data_client(const Message& msg, MsgBuf& buffer, size_t& raw_len);
};
} // namespace logproxy
} // namespace oceanbase

176
src/codec/legacy_decoder.cpp Normal file
View File

@ -0,0 +1,176 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <stdint.h>
#include "common/guard.hpp"
#include "communication/channel.h"
#include "codec/decoder.h"
#include "legacy.pb.h"
namespace oceanbase {
namespace logproxy {
static PacketError decode_v1(Channel& ch, Message*& message)
{
// V1 is protobuf handshake packet:
// [4] pb packet length
// [pb packet length] pb buffer
uint32_t payload_size = 0;
if (ch.readn((char*)(&payload_size), 4) != OMS_OK) {
OMS_ERROR << "Failed to read message payload size, ch:" << ch.peer().id() << ", error:" << strerror(errno);
return PacketError::NETWORK_ERROR;
}
payload_size = be_to_cpu(payload_size);
// FIXME.. use an mem pool
char* payload_buf = (char*)malloc(payload_size);
if (nullptr == payload_buf) {
OMS_ERROR << "Failed to malloc memory for message data. size:" << payload_size << ", ch:" << ch.peer().id();
return PacketError::OUT_OF_MEMORY;
}
FreeGuard<char*> payload_buf_guard(payload_buf);
if (ch.readn(payload_buf, payload_size) != 0) {
OMS_ERROR << "Failed to read message. ch:" << ch.peer().id() << ", error:" << strerror(errno);
return PacketError::NETWORK_ERROR;
}
payload_buf_guard.release();
legacy::PbPacket pb_packet;
bool ret = pb_packet.ParseFromArray(payload_buf, payload_size);
if (!ret) {
OMS_ERROR << "Failed to parse payload, ch:" << ch.peer().id();
return PacketError::PROTOCOL_ERROR;
}
if ((MessageType)pb_packet.type() != MessageType::HANDSHAKE_REQUEST_CLIENT) {
OMS_ERROR << "Invalid packet type:" << pb_packet.type() << ", ch:" << ch.peer().id();
return PacketError::PROTOCOL_ERROR;
}
legacy::ClientHandShake handshake;
ret = handshake.ParseFromString(pb_packet.payload());
if (!ret) {
OMS_ERROR << "Failed to parse handshake, ch:" << ch.peer().id();
return PacketError::PROTOCOL_ERROR;
}
ClientHandshakeRequestMessage* msg = new (std::nothrow) ClientHandshakeRequestMessage;
msg->log_type = handshake.log_type();
msg->ip = handshake.client_ip();
msg->id = handshake.client_id();
msg->version = handshake.client_version();
msg->configuration = handshake.configuration();
message = msg;
return PacketError::SUCCESS;
}
/*
* =========== Message Header ============
* [4] type
*/
PacketError LegacyDecoder::decode(Channel& ch, MessageVersion version, Message*& message)
{
OMS_DEBUG << "Legacy decode with, version: " << (int)version;
if (version == MessageVersion::V1) {
return decode_v1(ch, message);
}
// type
uint32_t type = -1;
if (ch.readn((char*)&type, 4) != OMS_OK) {
OMS_ERROR << "Failed to read message header, ch:" << ch.peer().id() << ", error:" << strerror(errno);
return PacketError::NETWORK_ERROR;
}
type = be_to_cpu(type);
if (!is_type_available(type)) {
OMS_ERROR << "Invalid packet type:" << type << ", ch:" << ch.peer().id();
return PacketError::PROTOCOL_ERROR;
}
OMS_DEBUG << "Legacy message type:" << type;
int ret = OMS_OK;
switch ((MessageType)type) {
case MessageType::HANDSHAKE_REQUEST_CLIENT:
ret = decode_handshake_request(ch, message);
break;
default:
// We don not care other request type as a server decoder
return PacketError::IGNORE;
}
return ret == OMS_OK ? PacketError::SUCCESS : PacketError::PROTOCOL_ERROR;
}
static int read_varstr(Channel& ch, std::string& val)
{
uint32_t len = 0;
if (ch.readn((char*)&len, 4) != OMS_OK) {
return OMS_FAILED;
}
len = be_to_cpu(len);
char* buf = (char*)malloc(len);
FreeGuard<char*> ff(buf);
if (ch.readn(buf, len) != OMS_OK) {
return OMS_FAILED;
}
val.assign(buf, len);
return OMS_OK;
}
/*
* [1] log type
* [4+varstr] Client IP
* [4+varstr] Client ID
* [4+varstr] Client Version
* [4+varstr] Configuration
*/
int LegacyDecoder::decode_handshake_request(Channel& ch, Message*& message)
{
ClientHandshakeRequestMessage* msg = new (std::nothrow) ClientHandshakeRequestMessage;
if (ch.readn((char*)&msg->log_type, 1) != OMS_OK) {
OMS_ERROR << "Failed to read message log_type, ch:" << ch.peer().id() << ", error:" << strerror(errno);
return OMS_FAILED;
}
OMS_DEBUG << "log type:" << (int)msg->log_type;
if (read_varstr(ch, msg->ip) != OMS_OK) {
OMS_ERROR << "Failed to read message Client IP, ch:" << ch.peer().id() << ", error:" << strerror(errno);
return OMS_FAILED;
}
OMS_DEBUG << "client ip: " << msg->ip;
if (read_varstr(ch, msg->id) != OMS_OK) {
OMS_ERROR << "Failed to read message Client IP, ch:" << ch.peer().id() << ", error:" << strerror(errno);
return OMS_FAILED;
}
OMS_DEBUG << "client id: " << msg->id;
if (read_varstr(ch, msg->version) != OMS_OK) {
OMS_ERROR << "Failed to read message Client IP, ch:" << ch.peer().id() << ", error:" << strerror(errno);
return OMS_FAILED;
}
OMS_DEBUG << "client version: " << msg->version;
if (read_varstr(ch, msg->configuration) != OMS_OK) {
OMS_ERROR << "Failed to read message Client IP, ch:" << ch.peer().id() << ", error:" << strerror(errno);
return OMS_FAILED;
}
OMS_DEBUG << "configuration: " << msg->configuration;
message = msg;
return OMS_OK;
}
} // namespace logproxy
} // namespace oceanbase

252
src/codec/legacy_encoder.cpp Normal file
View File

@ -0,0 +1,252 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "lz4.h"
#include "MsgHeader.h"
#include "common/config.h"
#include "common/guard.hpp"
#include "codec/encoder.h"
namespace oceanbase {
namespace logproxy {
static Config& _s_config = Config::instance();
static int compress_data(const RecordDataMessage& msg, MsgBuf& buffer, size_t& raw_len)
{
std::vector<std::pair<const char*, size_t>> ptrs;
ptrs.reserve(msg.count());
uint32_t total_size = 0;
for (size_t i = 0; i < msg.count(); ++i) {
auto& record = msg.records[i + msg.offset()];
size_t size = 0;
// got independ address
const char* logmsg_buf = record->getFormatedString(&size);
if (logmsg_buf == nullptr) {
OMS_ERROR << "Failed to serialize logmsg";
return OMS_FAILED;
}
if (_s_config.verbose_packet.val()) {
// const MsgHeader* header = (const MsgHeader*)(logmsg_buf);
// OMS_DEBUG << "Encode logmsg Header, type: " << header->m_msgType << ", version: " << header->m_version
// << ", size: " << header->m_size;
}
ptrs.emplace_back(logmsg_buf, size);
total_size += (size + 8);
}
char* raw = (char*)malloc(total_size);
FreeGuard<char*> fg_raw(raw);
if (raw == nullptr) {
OMS_ERROR << "Failed to allocate raw buffer to compress, size:" << total_size;
return OMS_FAILED;
}
int bound_size = LZ4_COMPRESSBOUND(total_size);
char* compressed = (char*)malloc(bound_size);
FreeGuard<char*> fg(compressed);
if (compressed == nullptr) {
OMS_ERROR << "Failed to allocate LZ4 bound buffer, size:" << bound_size;
return OMS_FAILED;
}
uint32_t idx = msg.idx;
size_t offset = 0;
for (auto& ptr : ptrs) {
size_t block_size = ptr.second;
uint32_t seq_be = cpu_to_be(idx++);
uint32_t size_be = cpu_to_be((uint32_t)block_size);
memcpy(raw + offset, &seq_be, 4);
memcpy(raw + offset + 4, &size_be, 4);
memcpy(raw + offset + 8, ptr.first, block_size);
offset += (block_size + 8);
}
int compressed_size = LZ4_compress_fast(raw, compressed, total_size, bound_size, 1);
if (compressed_size <= 0) {
OMS_ERROR << "Failed to compress logmsg, raw size:" << total_size << ", bound size:" << bound_size;
return OMS_FAILED;
}
if (_s_config.verbose.val()) {
OMS_DEBUG << "compress packet raw from size:" << total_size << " to compressed size:" << compressed_size;
}
uint32_t packet_len_be = cpu_to_be((uint32_t)compressed_size + 9);
uint32_t orginal_size_be = cpu_to_be(total_size);
uint32_t compressed_size_be = cpu_to_be((uint32_t)compressed_size);
char* buf = (char*)malloc(13);
memcpy(buf, &packet_len_be, 4);
memset(buf + 4, (uint8_t)CompressType::LZ4, 1);
memcpy(buf + 5, &orginal_size_be, 4);
memcpy(buf + 9, &compressed_size_be, 4);
buffer.push_back(buf, 13);
raw_len = total_size + 13;
// transfer ownership to Msgbuf
fg.release();
buffer.push_back(compressed, compressed_size);
return OMS_OK;
}
LegacyEncoder::LegacyEncoder()
{
/*
* [4] response code
* [1+varstr] Server IP
* [1+varstr] Server Version
*/
_funcs.emplace((int8_t)MessageType::HANDSHAKE_RESPONSE_CLIENT, [](const Message& in_msg, MsgBuf& buffer, size_t&) {
const ClientHandshakeResponseMessage& msg = (const ClientHandshakeResponseMessage&)in_msg;
size_t len = 4 + 1 + msg.server_ip.size() + 1 + msg.server_version.size();
char* buf = (char*)malloc(len);
if (buf == nullptr) {
OMS_ERROR << "Failed to encode handshake request due to failed to alloc memory";
return OMS_FAILED;
}
OMS_INFO << "Encode handshake response to send:" << msg.debug_string();
// Response code
memset(buf, 0, 4);
// Server IP
size_t offset = 4;
uint8_t varlen = msg.server_ip.size();
memcpy(buf + offset, &varlen, 1);
offset += 1;
memcpy(buf + offset, msg.server_ip.c_str(), varlen);
offset += varlen;
// Server version
varlen = msg.server_version.size();
memcpy(buf + offset, &varlen, 1);
offset += 1;
memcpy(buf + offset, msg.server_version.c_str(), varlen);
buffer.push_back(buf, len);
return OMS_OK;
});
/*
* [4] Packet len
* [1] Compress type
* [4] Original data len
* [4] Compressed data len
* [Compressed data len] Datas
* ===== for each message in Datas
* [4] Message ID
* [4] Message Len
* [Message Len] LogMsg
*/
_funcs.emplace((int8_t)MessageType::DATA_CLIENT, [](const Message& in_msg, MsgBuf& buffer, size_t& raw_len) {
const RecordDataMessage& msg = (const RecordDataMessage&)in_msg;
if (msg.compress_type == CompressType::LZ4) {
return compress_data(msg, buffer, raw_len);
}
uint32_t idx = msg.idx;
uint32_t total_size = 0;
for (size_t i = 0; i < msg.count(); ++i) {
auto& record = msg.records[i + msg.offset()];
size_t size = 0;
// got independ address
const char* logmsg_buf = record->getFormatedString(&size);
if (logmsg_buf == nullptr) {
OMS_ERROR << "Failed to serialize log record";
return OMS_FAILED;
}
if (_s_config.verbose_packet.val()) {
const MsgHeader* header = (const MsgHeader*)(logmsg_buf);
OMS_DEBUG << "Encode logmsg Header, type: " << header->m_msgType << ", version: " << header->m_version
<< ", size: " << header->m_size;
}
uint32_t seq_be = cpu_to_be(idx++);
uint32_t size_be = cpu_to_be((uint32_t)size);
buffer.push_back_copy((char*)&seq_be, 4);
buffer.push_back_copy((char*)&size_be, 4);
buffer.push_back((char*)logmsg_buf, size, false);
total_size += (size + 8);
}
uint32_t packet_len_be = cpu_to_be(total_size + 9);
total_size = cpu_to_be(total_size);
char* buf = (char*)malloc(4 + 1 + 4 + 4);
memcpy(buf, &packet_len_be, 4);
memset(buf + 4, (uint8_t)CompressType::PLAIN, 1);
memcpy(buf + 5, &total_size, 4);
memcpy(buf + 9, &total_size, 4);
buffer.push_front(buf, 13);
return OMS_OK;
});
_funcs.emplace((int8_t)MessageType::STATUS, [](const Message& in_msg, MsgBuf& buffer, size_t&) { return OMS_OK; });
/*
* [4] Reponse code
* [4+varstr] Error message
*/
_funcs.emplace((int8_t)MessageType::ERROR_RESPONSE, [](const Message& in_msg, MsgBuf& buffer, size_t&) {
const ErrorMessage& msg = (const ErrorMessage&)in_msg;
size_t len = 4 + 4 + msg.message.size();
char* buf = (char*)malloc(len);
if (buf == nullptr) {
OMS_ERROR << "Failed to encode error message due to failed to alloc memory";
return OMS_FAILED;
}
// Error message
uint32_t code_be = cpu_to_be((uint32_t)msg.code);
memcpy(buf, &code_be, 4);
uint32_t varlen_be = cpu_to_be((uint32_t)msg.message.size());
memcpy(buf + 4, &varlen_be, 4);
if (msg.message.size() != 0) {
memcpy(buf + 8, msg.message.c_str(), msg.message.size());
}
// buf's ownership transfered to buffer
buffer.push_back(buf, len);
return OMS_OK;
});
}
int LegacyEncoder::encode(const Message& msg, MsgBuf& buffer, size_t& raw_len)
{
int ret = _funcs[(int8_t)msg.type()](msg, buffer, raw_len);
if (ret == OMS_FAILED) {
return OMS_FAILED;
}
// append header
size_t len = 2 + 4;
char* buf = (char*)malloc(len);
// version code
memset(buf, 0, 2);
// response type code
uint32_t msg_type_be = cpu_to_be((uint32_t)msg.type());
memcpy(buf + 2, &msg_type_be, 4);
buffer.push_front(buf, len);
return ret;
}
} // namespace logproxy
} // namespace oceanbase

332
src/codec/message.cpp Normal file
View File

@ -0,0 +1,332 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "codec/message.h"
#include "codec/msg_buf.h"
#include "common/log.h"
#include "common/common.h"
#include "common/config.h"
#include "LogMsgFactory.h"
#include "LogRecord.h"
#include "MsgHeader.h"
#include "lz4.h"
namespace oceanbase {
namespace logproxy {
static Config& _s_config = Config::instance();
#ifndef NEED_MAPPING_CLASS
const std::string _s_logmsg_type = "LogRecordImpl";
#else
const std::string _s_logmsg_type = "BinlogRecordImpl";
#endif
bool is_version_available(uint16_t version_val)
{
return version_val >= 0 && version_val <= 2;
}
bool is_type_available(int8_t type_val)
{
return type_val >= -1 && type_val <= 8;
}
Message::Message(MessageType type) : _type(type)
{}
const std::string& Message::debug_string() const
{
switch (_type) {
case MessageType::HANDSHAKE_REQUEST_CLIENT:
return ((ClientHandshakeRequestMessage*)this)->to_string();
case MessageType::HANDSHAKE_RESPONSE_CLIENT:
return ((ClientHandshakeResponseMessage*)this)->to_string();
case MessageType::ERROR_RESPONSE:
return ((ErrorMessage*)this)->to_string();
case MessageType::DATA_CLIENT: {
static std::string msg = "DATA_CLIENT";
return msg;
}
case MessageType::STATUS: {
// TODO
static std::string msg = "RUNTIME STATUS";
return msg;
}
case MessageType::UNKNOWN:
default: {
static std::string msg = "UNKNOWN MESSAGE";
return msg;
}
}
}
ErrorMessage::ErrorMessage() : Message(MessageType::ERROR_RESPONSE)
{}
ErrorMessage::ErrorMessage(int code, const std::string& message)
: Message(MessageType::ERROR_RESPONSE), code(code), message(message)
{}
ClientHandshakeRequestMessage::ClientHandshakeRequestMessage() : Message(MessageType::HANDSHAKE_REQUEST_CLIENT)
{}
ClientHandshakeRequestMessage::ClientHandshakeRequestMessage(
int log_type, const char* ip, const char* id, const char* version, bool enable_monitor, const char* configuration)
: Message(MessageType::HANDSHAKE_REQUEST_CLIENT),
log_type(log_type),
ip(ip),
id(id),
version(version),
enable_monitor(enable_monitor),
configuration(configuration)
{}
ClientHandshakeResponseMessage::ClientHandshakeResponseMessage(
int in_code, const std::string& in_ip, const std::string& in_version)
: Message(MessageType::HANDSHAKE_RESPONSE_CLIENT), code(in_code), server_ip(in_ip), server_version(in_version)
{}
RuntimeStatusMessage::RuntimeStatusMessage() : Message(MessageType::STATUS)
{}
RuntimeStatusMessage::RuntimeStatusMessage(const char* ip, int port, int stream_count, int worker_count)
: Message(MessageType::STATUS), ip(ip), port(port), stream_count(stream_count), worker_count(worker_count)
{}
RecordDataMessage::RecordDataMessage(std::vector<ILogRecord*>& in_records)
: Message(MessageType::DATA_CLIENT), records(in_records), _offset(0), _count(in_records.size())
{}
RecordDataMessage::RecordDataMessage(std::vector<ILogRecord*>& in_records, size_t offset, size_t count)
: Message(MessageType::DATA_CLIENT), records(in_records), _offset(offset), _count(count)
{}
RecordDataMessage::~RecordDataMessage() = default;
int RecordDataMessage::encode_log_records(MsgBuf& buffer, size_t& raw_len) const
{
switch (compress_type) {
case CompressType::PLAIN: {
int ret = encode_log_records_plain(buffer);
if (ret == OMS_OK) {
raw_len = buffer.byte_size();
}
return ret;
}
case CompressType::LZ4: {
return encode_log_records_lz4(buffer, raw_len);
}
default: {
OMS_ERROR << "Unsupported compress type: " << (int)compress_type;
return OMS_FAILED;
}
}
}
int RecordDataMessage::encode_log_records_plain(MsgBuf& buffer) const
{
MsgBuf tmp_buffer;
for (size_t i = 0; i < _count; ++i) {
ILogRecord* log_record = records[i + _offset];
size_t size = 0;
// got independ address
const char* log_record_buffer = log_record->getFormatedString(&size);
if (nullptr == log_record_buffer) {
OMS_ERROR << "Failed to serialize log record";
return OMS_FAILED;
}
const MsgHeader* header = (const MsgHeader*)(log_record_buffer);
if (_s_config.verbose_packet.val()) {
OMS_DEBUG << "Encode LogMessage Header, type: " << header->m_msgType << ", version: " << header->m_version
<< ", size: " << header->m_size;
}
size_t calc_size = header->m_size + sizeof(MsgHeader);
if (calc_size != size) {
if (calc_size > size) {
OMS_FATAL << "LogMessage Invalid, header calc size:" << calc_size << " > buffer size:" << size;
return OMS_FAILED;
}
if (_s_config.verbose_packet.val()) {
OMS_WARN << "LogMessage header size:" << calc_size << " != toString size:" << size << ". adjust to header size";
}
size = calc_size;
}
// LogMessage buffer freed outside
tmp_buffer.push_back((char*)log_record_buffer, size, false);
}
buffer.swap(tmp_buffer);
return OMS_OK;
}
int RecordDataMessage::encode_log_records_lz4(MsgBuf& buffer, size_t& raw_len) const
{
int ret = encode_log_records_plain(buffer);
if (ret != OMS_OK) {
OMS_ERROR << "Failed to encode log records(plain) in lz4 mode";
return ret;
}
if (buffer.count() == 0) {
OMS_WARN << "No log records";
return OMS_OK;
}
raw_len = buffer.byte_size();
char* plain_buffer = nullptr;
if (buffer.count() == 1) {
plain_buffer = buffer.begin()->buffer();
} else {
plain_buffer = (char*)malloc(raw_len);
if (nullptr == plain_buffer) {
OMS_ERROR << "Failed to alloc memory. size=" << raw_len;
return OMS_FAILED;
}
MsgBufReader reader(buffer);
ret = reader.read(plain_buffer, raw_len);
if (ret != OMS_OK) {
OMS_ERROR << "Failed to read buffer. size=" << raw_len;
free(plain_buffer);
return ret;
}
}
const int compress_bound = LZ4_compressBound(raw_len);
char* compressed_buffer = (char*)malloc(compress_bound);
if (nullptr == compressed_buffer) {
OMS_ERROR << "Failed to alloc compressed buffer. size=" << compress_bound;
if (buffer.count() != 1) {
free(plain_buffer);
}
return OMS_FAILED;
}
const int compressed_size = LZ4_compress_default(plain_buffer, compressed_buffer, raw_len, compress_bound);
if (compressed_size <= 0) {
OMS_ERROR << "LZ4 compress failed. src size=" << raw_len << ", compressed bound=" << compress_bound
<< ", compress return=" << compressed_size;
if (buffer.count() != 1) {
free(plain_buffer);
}
free(compressed_buffer);
return OMS_FAILED;
}
if (buffer.count() != 1) {
free(plain_buffer);
}
OMS_DEBUG << "Encode client data success(lz4). raw_len=" << raw_len << ", compressed_size=" << compressed_size;
MsgBuf compressed_message_buffer;
compressed_message_buffer.push_back(compressed_buffer, compressed_size);
buffer.swap(compressed_message_buffer);
return OMS_OK;
}
int RecordDataMessage::decode_log_records(
CompressType in_compress_type, const char* buffer, size_t buffer_size, size_t raw_len, int expect_count)
{
compress_type = in_compress_type;
switch (compress_type) {
case CompressType::PLAIN: {
return decode_log_records_plain(buffer, buffer_size, expect_count);
}
case CompressType::LZ4: {
return decode_log_records_lz4(buffer, buffer_size, raw_len, expect_count);
}
default: {
OMS_ERROR << "Unsupported compress type: " << (int)compress_type;
return OMS_FAILED;
}
}
}
int RecordDataMessage::decode_log_records_plain(const char* buffer, size_t size, int expect_count)
{
if (nullptr == buffer || 0 == size) {
OMS_ERROR << "Invalid argument. buffer=" << (intptr_t)buffer << ", buffer_size=" << size;
return OMS_FAILED;
}
std::vector<ILogRecord*> log_records;
int count = 0;
size_t offset = 0;
while (offset < size) {
if (size - offset < sizeof(MsgHeader)) {
OMS_ERROR << "Buffer is not enough for a log record. size=" << size - offset;
return OMS_FAILED;
}
const MsgHeader* header = (const MsgHeader*)(buffer + offset);
if (_s_config.verbose_packet.val()) {
OMS_DEBUG << "Decode LogMessage Header, type: " << header->m_msgType << ", version: " << header->m_version
<< ", size: " << header->m_size;
}
uint32_t log_record_size = header->m_size + sizeof(MsgHeader);
ILogRecord* log_record = LogMsgFactory::createLogRecord(_s_logmsg_type, buffer + offset, log_record_size);
if (nullptr == log_record) {
OMS_ERROR << "Failed to create log record";
return OMS_FAILED;
}
log_records.push_back(log_record);
offset += log_record_size;
++count;
}
if (count != expect_count) {
OMS_ERROR << "Expect " << expect_count << " record, but " << log_records.size() << " parsed";
return OMS_FAILED;
}
OMS_DEBUG << "Total " << log_records.size() << " log records have been decoded from buffer with size: " << size;
records.swap(log_records);
_count = count;
return OMS_OK;
}
int RecordDataMessage::decode_log_records_lz4(const char* buffer, size_t buffer_size, size_t raw_size, int expect_count)
{
char* decompressed_buffer = (char*)malloc(raw_size);
if (nullptr == decompressed_buffer) {
OMS_ERROR << "Failed to alloc memory. count=" << raw_size;
return OMS_FAILED;
}
size_t decompressed_size = LZ4_decompress_safe(buffer, decompressed_buffer, buffer_size, raw_size);
if (decompressed_size != raw_size) {
OMS_ERROR << "Failed to decompress log record buffer. compressed_size=" << buffer_size << ". raw_len=" << raw_size
<< ". lz4 decompres return=" << decompressed_size;
free(decompressed_buffer);
return OMS_FAILED;
}
OMS_DEBUG << "Decompress log record buffer success";
int ret = decode_log_records_plain(decompressed_buffer, raw_size, expect_count);
if (ret != OMS_OK) {
OMS_ERROR << "Failed to decode log record(plain)";
}
free(decompressed_buffer);
return ret;
}
} // namespace logproxy
} // namespace oceanbase

262
src/codec/message.h Normal file
View File

@ -0,0 +1,262 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <string>
#include <vector>
#include "LogRecord.h"
#include "common/model.h"
namespace oceanbase {
namespace logproxy {
extern const std::string _s_logmsg_type;
#ifndef NEED_MAPPING_CLASS
using namespace oceanbase::logmessage;
#endif
enum class MessageVersion : uint16_t {
V0 = 0,
V1 = 1,
V2 = 2,
};
bool is_version_available(uint16_t version_val);
enum class MessageType : int8_t {
ERROR_RESPONSE = -1,
UNKNOWN = 0,
HANDSHAKE_REQUEST_CLIENT = 1,
HANDSHAKE_RESPONSE_CLIENT = 2,
// HANDSHAKE_REQUEST_LOGREADER = 3,
// HANDSHAKE_RESPONSE_LOGREADER = 4,
// DATA_LOGREADER = 5,
DATA_CLIENT = 6,
STATUS = 7,
// STATUS_LOGREADER = 8,
};
bool is_type_available(int8_t type_val);
enum class CompressType {
PLAIN = 0,
LZ4 = 1,
};
enum class PacketError {
SUCCESS,
IGNORE,
OUT_OF_MEMORY,
PROTOCOL_ERROR,
NETWORK_ERROR,
};
enum ErrorCode {
////////// 0~499: process error ////////////
/**
* general error
*/
NONE = 0,
/**
* inner error
*/
E_INNER = 1,
/**
* failed to connect
*/
E_CONNECT = 2,
/**
* exceed max retry connect count
*/
E_MAX_RECONNECT = 3,
/**
* user callback throws exception
*/
E_USER = 4,
////////// 500~: recv data error ////////////
/**
* unknown data protocol
*/
E_PROTOCOL = 500,
/**
* unknown header type
*/
E_HEADER_TYPE = 501,
/**
* failed to auth
*/
NO_AUTH = 502,
/**
* unknown compress type
*/
E_COMPRESS_TYPE = 503,
/**
* length not match
*/
E_LEN = 504,
/**
* failed to parse data
*/
E_PARSE = 505
};
constexpr char PACKET_MAGIC[] = {'x', 'i', '5', '3', 'g', ']', 'q'};
constexpr size_t PACKET_MAGIC_SIZE = sizeof(PACKET_MAGIC);
const size_t PACKET_VERSION_SIZE = 2;
class Message {
public:
explicit Message(MessageType message_type);
virtual ~Message() = default;
inline MessageType type() const
{
return _type;
}
inline MessageVersion version() const
{
return _version;
}
void set_version(MessageVersion version)
{
_version = version;
}
virtual uint64_t size()
{
return 0;
}
virtual const std::string& debug_string() const;
protected:
MessageType _type;
MessageVersion _version = MessageVersion::V2;
};
class ErrorMessage : public Message, public Model {
public:
ErrorMessage();
ErrorMessage(int code, const std::string& message);
private:
OMS_MF(int, code);
OMS_MF(std::string, message);
};
class ClientHandshakeRequestMessage : public Message, public Model {
public:
ClientHandshakeRequestMessage();
ClientHandshakeRequestMessage(int log_type, const char* ip, const char* id, const char* version, bool enable_monitor,
const char* configuration);
~ClientHandshakeRequestMessage() override = default;
OMS_MF_DFT(uint8_t, log_type, 0);
OMS_MF(std::string, id);
OMS_MF(std::string, ip);
OMS_MF(std::string, version);
OMS_MF(std::string, configuration);
OMS_MF_DFT(bool, enable_monitor, false);
};
class ClientHandshakeResponseMessage : public Message, public Model {
public:
ClientHandshakeResponseMessage(int code, const std::string& in_ip, const std::string& in_version);
~ClientHandshakeResponseMessage() override = default;
private:
OMS_MF_DFT(int, code, -1);
OMS_MF(std::string, server_ip);
OMS_MF(std::string, server_version);
};
class RuntimeStatusMessage : public Message, public Model {
public:
RuntimeStatusMessage();
RuntimeStatusMessage(const char* ip, int port, int stream_count, int worker_count);
~RuntimeStatusMessage() override = default;
private:
OMS_MF(std::string, ip);
OMS_MF_DFT(int, port, -1);
OMS_MF_DFT(int, stream_count, -1);
OMS_MF_DFT(int, worker_count, -1);
};
class MsgBuf;
class RecordDataMessage : public Message {
public:
~RecordDataMessage() override;
explicit RecordDataMessage(std::vector<ILogRecord*>& records);
RecordDataMessage(std::vector<ILogRecord*>& records, size_t offset, size_t count);
inline size_t offset() const
{
return _offset;
}
inline size_t count() const
{
return _count;
}
int encode_log_records(MsgBuf& buffer, size_t& raw_len) const;
int decode_log_records(CompressType compress_type, const char* buffer, size_t size, size_t raw_len, int expect_count);
protected:
int decode_log_records_plain(const char* buffer, size_t size, int expect_count);
int decode_log_records_lz4(const char* buffer, size_t size, size_t raw_size, int expect_count);
int encode_log_records_plain(MsgBuf& buffer) const;
int encode_log_records_lz4(MsgBuf& buffer, size_t& raw_len) const;
public:
CompressType compress_type = CompressType::PLAIN;
std::vector<ILogRecord*>& records;
size_t _offset = 0;
size_t _count = 0;
// index to count message seq for a client session
uint32_t idx = 0;
};
} // namespace logproxy
} // namespace oceanbase

237
src/codec/msg_buf.cpp Normal file
View File

@ -0,0 +1,237 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <string.h>
#include "codec/msg_buf.h"
#include "codec/codec_endian.h"
#include "common/log.h"
namespace oceanbase {
namespace logproxy {
size_t MsgBuf::byte_size() const
{
size_t result = 0;
for (auto& chunk : _chunks) {
result += chunk.size();
}
return result;
}
////////////////////////////////////////////////////////////////////////////////
MsgBufReader::MsgBufReader(const MsgBuf& buffer) : _buffer(buffer)
{
_iter = buffer.begin();
_byte_size = _buffer.byte_size();
}
int MsgBufReader::read(char* buf, size_t size)
{
if (size == 0) {
return 0;
}
if (buf == nullptr) {
OMS_ERROR << "Invalid argument. buffer=" << (intptr_t)buf << ", size=" << size;
return -1;
}
return read(buf, size, false);
}
int MsgBufReader::read(char* buf, size_t size, bool skip)
{
auto chunk_iter = _iter;
size_t chunk_pos = _pos;
size_t read_size = 0;
for (; read_size < size && chunk_iter != _buffer.end();) {
const MsgBuf::Chunk& chunk = *chunk_iter;
const size_t chunk_size = chunk.size() - chunk_pos;
if (read_size + chunk_size >= size) {
if (!skip) {
memcpy(buf + read_size, chunk.buffer() + chunk_pos, size - read_size);
}
chunk_pos += size - read_size;
read_size = size;
_iter = chunk_iter;
_pos = chunk_pos;
_read_size += size;
return 0;
} else {
if (!skip) {
memcpy(buf + read_size, chunk.buffer() + chunk_pos, chunk_size);
}
read_size += chunk_size;
++chunk_iter;
chunk_pos = 0;
}
}
OMS_ERROR << "Size read: " << read_size << " not enough, expected size: " << size;
return -1;
}
int MsgBufReader::read_uint8(uint8_t& i)
{
return read((char*)&i, sizeof(i));
}
int MsgBufReader::read_uint16(uint16_t& i)
{
return read((char*)&i, sizeof(i));
}
int MsgBufReader::read_uint24(uint32_t& i)
{
return read((char*)&i, 3);
}
int MsgBufReader::read_uint32(uint32_t& i)
{
return read((char*)&i, sizeof(i));
}
int MsgBufReader::read_uint48(uint64_t& i)
{
return read((char*)&i, 6);
}
int MsgBufReader::read_uint64(uint64_t& i)
{
return read((char*)&i, sizeof(i));
}
int MsgBufReader::next(const char** buffer, int* size)
{
if (_iter == _buffer.end()) {
OMS_DEBUG << "got EOF while call next";
return -1;
}
do {
auto& chunk = *_iter;
*buffer = chunk.buffer() + _pos;
*size = chunk.size() - _pos;
++_iter;
_pos = 0;
} while (*size == 0 && _iter != _buffer.end());
if (*size == 0 && _iter == _buffer.end()) {
OMS_DEBUG << "Touch the end of buffer";
return -1;
}
return 0;
}
int MsgBufReader::backward(size_t count)
{
if (count > _read_size) {
OMS_ERROR << "Failed to backward data. count=" << count << ", read_size=" << _read_size;
return -1;
}
size_t backward_count = count;
auto chunk_iter = _iter;
size_t chunk_pos = _pos;
while (backward_count > chunk_pos) {
backward_count -= chunk_pos;
--chunk_iter;
chunk_pos = chunk_iter->size();
}
if (backward_count > 0) {
chunk_pos -= backward_count;
}
_iter = chunk_iter;
_pos = chunk_pos;
_read_size -= count;
return 0;
}
int MsgBufReader::forward(size_t count)
{
return read(nullptr, count, true);
}
size_t MsgBufReader::read_size() const
{
return _read_size;
}
size_t MsgBufReader::byte_size() const
{
return _byte_size;
}
size_t MsgBufReader::remain_size() const
{
return _byte_size - _read_size;
}
bool MsgBufReader::has_more() const
{
auto iter = _iter;
size_t pos = _pos;
while (iter != _buffer.end()) {
auto& chunk = *iter;
if (pos < chunk.size()) {
return true;
}
++iter;
pos = 0;
}
return false;
}
std::string MsgBufReader::debug_info() const
{
LogStream ls(0, "", 0, nullptr);
ls << "[MsgBuf] all: " << byte_size() << ", "
<< "read: " << read_size() << ", "
<< "remain: " << remain_size();
return ls.str();
}
int MysqlBufReader::read_lenenc_uint(uint64_t& value)
{
value = 0;
uint8_t byte = 0;
read((char*)&byte, 1);
if (byte < 251) {
value = byte;
} else if (byte == 252) {
read((char*)&value, 2);
} else if (byte == 253) {
read((char*)&value, 3);
} else if (byte == 254) {
read((char*)&value, 8);
} else if (byte == 251) {
return 251;
} else if (byte == 255) {
return 255;
}
return 0;
}
void MysqlBufReader::read_lenenc_str(std::string& value)
{
uint64_t length = 0;
read_lenenc_uint(length);
// FIXME... length secure check
value.resize(length);
read((char*)value.data(), length);
}
} // namespace logproxy
} // namespace oceanbase

197
src/codec/msg_buf.h Normal file
View File

@ -0,0 +1,197 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <stdlib.h>
#include <deque>
#include "common/log.h"
#include "codec/codec_endian.h"
namespace oceanbase {
namespace logproxy {
class MsgBuf {
public:
class Chunk {
public:
Chunk(char* buffer, size_t size) : _buffer(buffer), _size(size)
{}
Chunk(char* buffer, size_t size, bool owned) : _buffer(buffer), _size(size), _owned(owned)
{}
Chunk(Chunk&& other) noexcept
{
_buffer = other._buffer;
_size = other._size;
_owned = other._owned;
other._owned = false;
}
~Chunk()
{
if (_owned && _buffer != nullptr) {
free(_buffer);
_buffer = nullptr;
}
}
char* buffer() const
{
return _buffer;
}
size_t size() const
{
return _size;
}
private:
char* _buffer = nullptr;
size_t _size = 0;
bool _owned = true;
};
public:
MsgBuf() = default;
~MsgBuf() = default;
void reset()
{
_chunks.clear();
}
void swap(MsgBuf& other)
{
if (&other != this) {
this->_chunks.swap(other._chunks);
}
}
void push_back(char* buffer, size_t size, bool owned = true)
{
_chunks.emplace_back(buffer, size, owned);
}
void push_back_copy(char* buffer, size_t size)
{
char* nbuf = static_cast<char*>(malloc(size));
memcpy(nbuf, buffer, size);
_chunks.emplace_back(nbuf, size, true);
}
void push_front(char* buffer, int size, bool owned = true)
{
_chunks.emplace_front(buffer, size, owned);
}
size_t count() const
{
return _chunks.size();
}
std::deque<Chunk>::const_iterator begin() const
{
return _chunks.begin();
}
std::deque<Chunk>::const_iterator end() const
{
return _chunks.end();
}
size_t byte_size() const;
private:
std::deque<Chunk> _chunks;
};
// TODO messageBufferWriter
class MsgBufReader {
public:
explicit MsgBufReader(const MsgBuf& buffer);
int read(char* buf, size_t size);
/**
* Read data into buffer. If it is not enough, reading position is untouched
* @param buf target
* @param size size to read, in byte
* @param skip true: no reading(memcpy), just skip it
*/
int read(char* buf, size_t size, bool skip);
int read_uint8(uint8_t& i);
int read_uint16(uint16_t& i);
int read_uint24(uint32_t& i);
int read_uint32(uint32_t& i);
int read_uint48(uint64_t& i);
int read_uint64(uint64_t& i);
template <class Integer>
int read_int(Integer& i)
{
int ret = read((char*)&i, sizeof(i));
if (ret != 0) {
return ret;
}
i = le_to_cpu(i);
return 0;
}
int next(const char** buf, int* size);
int backward(size_t count);
int forward(size_t count);
size_t read_size() const;
/**
* Total size of MessageBuffer in byte
*/
size_t byte_size() const;
size_t remain_size() const;
bool has_more() const;
std::string debug_info() const;
private:
const MsgBuf& _buffer;
std::deque<MsgBuf::Chunk>::const_iterator _iter;
size_t _pos = 0;
size_t _byte_size = 0;
size_t _read_size = 0;
};
class MysqlBufReader : public MsgBufReader {
public:
explicit MysqlBufReader(const MsgBuf& buffer) : MsgBufReader(buffer)
{}
int read_lenenc_uint(uint64_t& value);
void read_lenenc_str(std::string& value);
};
} // namespace logproxy
} // namespace oceanbase

265
src/codec/protobuf_decoder.cpp Normal file
View File

@ -0,0 +1,265 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "google/protobuf/io/zero_copy_stream.h"
#include "common/guard.hpp"
#include "common/config.h"
#include "communication/channel.h"
#include "codec/decoder.h"
#include "logproxy.pb.h"
namespace oceanbase {
namespace logproxy {
static Config& _s_config = Config::instance();
/*
* =========== Message Header ============
* [1] type
* [4] packet size
* [packet_size] pb payload
*/
const size_t MESSAGE_HEADER_SIZE_V2 = 1 + 4;
PacketError ProtobufDecoder::decode(Channel& ch, MessageVersion version, Message*& message)
{
char header_buf[MESSAGE_HEADER_SIZE_V2];
if (ch.readn(header_buf, MESSAGE_HEADER_SIZE_V2) != OMS_OK) {
OMS_ERROR << "Failed to read message header, ch:" << ch.peer().id() << ", error:" << strerror(errno);
return PacketError::NETWORK_ERROR;
}
// type
int8_t type = -1;
memcpy(&type, header_buf, 1);
if (!is_type_available(type)) {
OMS_ERROR << "Invalid packet type:" << type << ", ch:" << ch.peer().id();
return PacketError::PROTOCOL_ERROR;
}
// payload size
uint32_t payload_size = 0;
memcpy(&payload_size, header_buf + 1, 4);
payload_size = be_to_cpu(payload_size);
// TODO... suppose that no large message
if (payload_size > Config::instance().max_packet_bytes.val()) {
OMS_ERROR << "Too large message size: " << payload_size
<< ", exceed max: " << Config::instance().max_packet_bytes.val();
return PacketError::PROTOCOL_ERROR;
}
// FIXME.. use an mem pool
char* payload_buf = (char*)malloc(payload_size);
if (nullptr == payload_buf) {
OMS_ERROR << "Failed to malloc memory for message data. size:" << payload_size << ", ch:" << ch.peer().id();
return PacketError::OUT_OF_MEMORY;
}
FreeGuard<char*> payload_buf_guard(payload_buf);
if (ch.readn(payload_buf, payload_size) != 0) {
OMS_ERROR << "Failed to read message. ch:" << ch.peer().id() << ", error:" << strerror(errno);
return PacketError::NETWORK_ERROR;
}
payload_buf_guard.release();
MsgBuf buffer;
buffer.push_back(payload_buf, payload_size);
// Transfer buffer owner to message_buffer
// buffer's memory will freed by MessageBuffer
int ret = decode_payload((MessageType)type, buffer, message);
if (ret != OMS_OK) {
OMS_ERROR << "Failed to decode_payload message, ch:" << ch.peer().id();
return PacketError::PROTOCOL_ERROR;
}
return PacketError::SUCCESS;
}
int ProtobufDecoder::decode_payload(MessageType type, const MsgBuf& buffer, Message*& msg)
{
MsgBufReader reader(buffer);
switch ((MessageType)type) {
case MessageType::HANDSHAKE_REQUEST_CLIENT: {
return decode_handshake_request(reader, msg);
}
case MessageType::HANDSHAKE_RESPONSE_CLIENT: {
return decode_handshake_response(reader, msg);
}
case MessageType::STATUS: {
return decode_runtime_status(reader, msg);
}
case MessageType::DATA_CLIENT: {
return decode_data_client(reader, msg);
}
default: {
OMS_ERROR << "Unknown message type: " << (int)type;
} break;
}
// should not go here
return OMS_FAILED;
}
class ZeroCopyStreamAdapter : public ::google::protobuf::io::ZeroCopyInputStream {
public:
explicit ZeroCopyStreamAdapter(MsgBufReader& buffer_reader) : _buffer_reader(buffer_reader)
{}
~ZeroCopyStreamAdapter() override = default;
bool Next(const void** data, int* size) override
{
return 0 == _buffer_reader.next((const char**)data, size);
}
void BackUp(int count) override
{
_buffer_reader.backward(count);
}
bool Skip(int count) override
{
return 0 == _buffer_reader.forward(count);
}
::google::protobuf::int64 ByteCount() const override
{
return _buffer_reader.read_size();
}
private:
MsgBufReader& _buffer_reader;
};
int ProtobufDecoder::decode_handshake_request(MsgBufReader& buffer_reader, Message*& out_msg)
{
ZeroCopyStreamAdapter zero_copy_stream(buffer_reader);
ClientHandshakeRequest pb_msg;
bool result = pb_msg.ParseFromZeroCopyStream(&zero_copy_stream);
if (!result) {
OMS_ERROR << "Failed to parse protobuf message from buffer";
return OMS_FAILED;
}
ClientHandshakeRequestMessage* msg = new (std::nothrow) ClientHandshakeRequestMessage((int)pb_msg.log_type(),
pb_msg.ip().c_str(),
pb_msg.id().c_str(),
pb_msg.version().c_str(),
pb_msg.enable_monitor(),
pb_msg.configuration().c_str());
if (nullptr == msg) {
OMS_ERROR << "Failed to create client_hand_shake_request_message.";
return OMS_FAILED;
}
out_msg = msg;
return OMS_OK;
}
int ProtobufDecoder::decode_handshake_response(MsgBufReader& buffer_reader, Message*& msg)
{
ZeroCopyStreamAdapter zero_copy_stream(buffer_reader);
ClientHandshakeResponse pb_msg;
bool result = pb_msg.ParseFromZeroCopyStream(&zero_copy_stream);
if (!result) {
OMS_ERROR << "Failed to parse protobuf message from buffer";
return OMS_FAILED;
}
// copy field
ClientHandshakeResponseMessage* response_msg = new (std::nothrow)
ClientHandshakeResponseMessage((int)pb_msg.code(), pb_msg.ip().c_str(), pb_msg.version().c_str());
if (nullptr == response_msg) {
OMS_ERROR << "Failed to create ClientHandShakeResponseMessage.";
return OMS_FAILED;
}
msg = response_msg;
return OMS_OK;
}
int ProtobufDecoder::decode_runtime_status(MsgBufReader& buffer_reader, Message*& _msg)
{
ZeroCopyStreamAdapter zero_copy_stream(buffer_reader);
RuntimeStatus pb_msg;
bool result = pb_msg.ParseFromZeroCopyStream(&zero_copy_stream);
if (!result) {
OMS_ERROR << "Failed to parse protobuf message from buffer";
return OMS_FAILED;
}
RuntimeStatusMessage* msg = new (std::nothrow) RuntimeStatusMessage(
pb_msg.ip().c_str(), (int)pb_msg.port(), (int)pb_msg.stream_count(), (int)pb_msg.worker_count());
if (nullptr == msg) {
OMS_ERROR << "Failed to create RuntimeStatusMessage.";
return OMS_FAILED;
}
_msg = msg;
return OMS_OK;
}
int ProtobufDecoder::decode_data_client(MsgBufReader& buffer_reader, Message*& _msg)
{
RecordData pb_msg;
// bool ret = pb_msg.ParseFromArray(str.c_str(), size);
ZeroCopyStreamAdapter zero_copy_stream(buffer_reader);
bool ret = pb_msg.ParseFromZeroCopyStream(&zero_copy_stream);
if (!ret) {
OMS_ERROR << "Failed to parse protobuf message from buffer";
return OMS_FAILED;
}
std::vector<ILogRecord*> records;
RecordDataMessage* msg = new (std::nothrow) RecordDataMessage(records);
if (nullptr == msg) {
OMS_ERROR << "Failed to create RecordDataMessage.";
return OMS_FAILED;
}
ret = msg->decode_log_records((CompressType)pb_msg.compress_type(),
pb_msg.records().data(),
pb_msg.records().size(),
pb_msg.raw_len(),
pb_msg.count());
if (ret != OMS_OK) {
OMS_ERROR << "Failed to decode log record";
delete msg;
return ret;
}
ILogRecord* record = msg->records[msg->offset()];
if (_s_config.verbose_packet.val()) {
OMS_INFO << "Fetched record from LogProxy, "
<< "compress type: " << pb_msg.compress_type() << ","
<< "raw_len: " << pb_msg.raw_len() << ","
<< "compressed_len: " << pb_msg.compressed_len() << ","
<< "count: " << pb_msg.count() << ","
<< "records size: " << pb_msg.records().size() << ","
<< "record_type: " << record->recordType() << ","
<< "timestamp: " << record->getTimestamp() << ","
<< "checkpoint: " << record->getFileNameOffset() << ","
<< "dbname: " << record->dbname() << ","
<< "tbname: " << record->tbname();
}
_msg = msg;
return OMS_OK;
}
} // namespace logproxy
} // namespace oceanbase

191
src/codec/protobuf_encoder.cpp Normal file
View File

@ -0,0 +1,191 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "google/protobuf/message.h"
#include "common/log.h"
#include "common/common.h"
#include "common/config.h"
#include "codec/message.h"
#include "codec/msg_buf.h"
#include "codec/codec_endian.h"
#include "codec/encoder.h"
#include "logproxy.pb.h"
namespace oceanbase {
namespace logproxy {
static const size_t PB_PACKET_HEADER_SIZE = PACKET_VERSION_SIZE + 1 /*type*/ + 4 /*packet size */;
static const size_t PB_PACKET_HEADER_SIZE_MAGIC = PACKET_MAGIC_SIZE + PB_PACKET_HEADER_SIZE;
int ProtobufEncoder::encode(const Message& msg, MsgBuf& buffer, size_t& raw_len)
{
switch (msg.type()) {
case MessageType::ERROR_RESPONSE: {
return encode_error_response(msg, buffer);
}
case MessageType::HANDSHAKE_REQUEST_CLIENT: {
return encode_client_handshake_request(msg, buffer);
}
case MessageType::HANDSHAKE_RESPONSE_CLIENT: {
return encode_client_handshake_response(msg, buffer);
}
case MessageType::STATUS: {
return encode_runtime_status(msg, buffer, raw_len);
}
case MessageType::DATA_CLIENT: {
return encode_data_client(msg, buffer, raw_len);
}
default: {
OMS_ERROR << "Unknown message type: " << (int)msg.type();
return OMS_FAILED;
}
}
}
static char* encode_message_header(MessageType type, int packet_size, bool magic)
{
size_t header_len = magic ? PB_PACKET_HEADER_SIZE_MAGIC : PB_PACKET_HEADER_SIZE;
char* buffer = (char*)malloc(header_len);
if (nullptr == buffer) {
OMS_ERROR << "Failed to alloc memory for message header. size=" << header_len;
return nullptr;
}
int offset = 0;
if (magic) {
memcpy(buffer, PACKET_MAGIC, sizeof(PACKET_MAGIC));
offset = sizeof(PACKET_MAGIC);
}
uint16_t version = cpu_to_be((uint16_t)MessageVersion::V2);
memcpy(buffer + offset, &version, sizeof(version));
offset += sizeof(version);
int8_t message_type = (int8_t)type;
memcpy(buffer + offset, &message_type, sizeof(message_type));
offset += sizeof(message_type);
uint32_t pb_packet_size = cpu_to_be((uint32_t)packet_size);
memcpy(buffer + offset, &pb_packet_size, sizeof(pb_packet_size));
return buffer;
}
int ProtobufEncoder::encode_message(
const google::protobuf::Message& pb_msg, MessageType type, MsgBuf& buffer, bool magic)
{
const size_t serialize_size = pb_msg.ByteSizeLong();
// TODO max message size
char* data_buffer = (char*)malloc(serialize_size);
if (nullptr == data_buffer) {
OMS_ERROR << "Failed to alloc memory. size=" << serialize_size;
return OMS_FAILED;
}
bool result = pb_msg.SerializeToArray(data_buffer, serialize_size);
if (!result) {
OMS_ERROR << "Failed to serialize protobuf message. size=" << serialize_size;
free(data_buffer);
return OMS_FAILED;
}
// Md5 md5(data_buffer, serialize_size);
// OMS_INFO << "Serialized protobuf message, size: " << serialize_size << ", MD5: " << md5.done();
char* header_buffer = encode_message_header(type, (int)serialize_size, magic);
if (nullptr == header_buffer) {
OMS_ERROR << "Failed to encode client_hand_shake_request 's message header";
free(data_buffer);
return OMS_FAILED;
}
buffer.push_back(header_buffer, magic ? PB_PACKET_HEADER_SIZE_MAGIC : PB_PACKET_HEADER_SIZE);
buffer.push_back(data_buffer, serialize_size);
return OMS_OK;
}
int ProtobufEncoder::encode_error_response(const Message& msg, MsgBuf& buffer)
{
const ErrorMessage oms_msg = (const ErrorMessage&)msg;
ErrorResponse pb_msg;
pb_msg.set_code(oms_msg.code);
pb_msg.set_message(oms_msg.message);
return encode_message(pb_msg, oms_msg.type(), buffer, false);
}
int ProtobufEncoder::encode_client_handshake_request(const Message& msg, MsgBuf& buffer)
{
const ClientHandshakeRequestMessage& request_message = (const ClientHandshakeRequestMessage&)msg;
ClientHandshakeRequest pb_msg;
pb_msg.set_log_type(request_message.log_type);
pb_msg.set_ip(request_message.ip);
pb_msg.set_id(request_message.id);
pb_msg.set_version(request_message.version);
pb_msg.set_enable_monitor(request_message.enable_monitor);
pb_msg.set_configuration(request_message.configuration);
return encode_message(pb_msg, request_message.type(), buffer, true);
}
int ProtobufEncoder::encode_client_handshake_response(const Message& msg, MsgBuf& buffer)
{
const ClientHandshakeResponseMessage& response_message = (const ClientHandshakeResponseMessage&)msg;
ClientHandshakeResponse pb_msg;
pb_msg.set_code(response_message.code);
pb_msg.set_ip(response_message.server_ip);
pb_msg.set_version(response_message.server_version);
return encode_message(pb_msg, response_message.type(), buffer, false);
}
int ProtobufEncoder::encode_runtime_status(const Message& msg, MsgBuf& buffer, size_t& raw_len)
{
const RuntimeStatusMessage& runtime_status_message = (const RuntimeStatusMessage&)msg;
RuntimeStatus pb_msg;
pb_msg.set_ip(runtime_status_message.ip);
pb_msg.set_port(runtime_status_message.port);
pb_msg.set_stream_count(runtime_status_message.stream_count);
pb_msg.set_worker_count(runtime_status_message.worker_count);
int ret = encode_message(pb_msg, runtime_status_message.type(), buffer, false);
raw_len = buffer.byte_size();
return ret;
}
int ProtobufEncoder::encode_data_client(const Message& msg, MsgBuf& buffer, size_t& raw_len)
{
RecordDataMessage& record_data_message = (RecordDataMessage&)msg;
MsgBuf records_buffer;
int ret = record_data_message.encode_log_records(records_buffer, raw_len);
if (ret != OMS_OK) {
OMS_ERROR << "Failed to encode log records. ret=" << ret;
return ret;
}
MsgBufReader records_buffer_reader(records_buffer);
const size_t records_size = records_buffer_reader.byte_size();
std::string pb_record_string(records_size, 0);
// FIXME... big data packet copy here due to perf laging
ret = records_buffer_reader.read((char*)pb_record_string.c_str(), records_size);
if (ret != OMS_OK) {
OMS_ERROR << "Failed to read buffer from records buffer, size=" << records_size;
return OMS_FAILED;
}
RecordData pb_msg;
pb_msg.set_records(pb_record_string);
pb_msg.set_compress_type((int)record_data_message.compress_type);
pb_msg.set_raw_len(raw_len);
pb_msg.set_compressed_len(records_size);
pb_msg.set_count(record_data_message.count());
return encode_message(pb_msg, record_data_message.type(), buffer, false);
}
} // namespace logproxy
} // namespace oceanbase

128
src/common/blocking_queue.hpp Normal file
View File

@ -0,0 +1,128 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <deque>
#include <thread>
#include <mutex>
#include <condition_variable>
namespace oceanbase {
namespace logproxy {
template <typename T>
class BlockingQueue {
public:
explicit BlockingQueue(const size_t max_queue_size = S_DEFAULT_MAX_QUEUE_SIZE) : _max_queue_size(max_queue_size)
{
// nothing to do
}
bool offer(const T& element, uint64_t timeout_us)
{
std::unique_lock<std::mutex> op_lock(_op_mutex);
while (_queue.size() >= _max_queue_size) {
std::cv_status st = _not_full.wait_for(op_lock, std::chrono::microseconds(timeout_us));
if (st == std::cv_status::timeout) {
return false;
}
}
if (_queue.size() >= _max_queue_size) {
return false;
}
_queue.emplace_back(element);
_not_empty.notify_one();
return true;
}
bool poll(T& element, uint64_t timeout_us)
{
std::unique_lock<std::mutex> op_lock(_op_mutex);
while (_queue.empty()) {
std::cv_status st = _not_empty.wait_for(op_lock, std::chrono::microseconds(timeout_us));
if (st == std::cv_status::timeout) {
return false;
}
}
if (_queue.empty()) {
return false;
}
element = _queue.front();
_queue.pop_front();
_not_full.notify_one();
return true;
}
bool poll(std::vector<T>& elements, uint64_t timeout_us)
{
std::unique_lock<std::mutex> op_lock(_op_mutex);
while (_queue.empty()) {
std::cv_status st = _not_empty.wait_for(op_lock, std::chrono::microseconds(timeout_us));
if (st == std::cv_status::timeout) {
return false;
}
}
if (_queue.empty()) {
return false;
}
while (!_queue.empty() && elements.size() < elements.capacity()) {
elements.push_back(_queue.front());
_queue.pop_front();
}
_not_full.notify_one();
return true;
}
size_t size(bool safe = true)
{
if (safe) {
std::lock_guard<std::mutex> lock(_op_mutex);
return _queue.size();
} else {
return _queue.size();
}
}
void clear()
{
clear([] {});
}
void clear(std::function<void(T&)> foreach)
{
std::lock_guard<std::mutex> lock(_op_mutex);
while (!_queue.empty()) {
if (foreach) {
foreach (_queue.front())
;
}
_queue.pop_front();
}
}
private:
static const size_t S_DEFAULT_MAX_QUEUE_SIZE = 60000;
size_t _max_queue_size;
std::deque<T> _queue;
std::mutex _op_mutex;
std::condition_variable _not_empty;
std::condition_variable _not_full;
};
} // namespace logproxy
} // namespace oceanbase

143
src/common/common.cpp Normal file
View File

@ -0,0 +1,143 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <errno.h>
#include <unistd.h>
#include <netdb.h>
#include <arpa/inet.h>
#include <string.h>
#include <string>
#include "common.h"
namespace oceanbase {
namespace logproxy {
static char* localhost()
{
static char* buf = nullptr;
static size_t buf_len = 0;
int myerror = 0;
if (!buf) {
do {
errno = 0;
if (buf) {
buf_len += buf_len;
char* buftmp = static_cast<char*>(realloc(buf, buf_len));
if (buftmp) {
buf = buftmp;
} else {
free(buf);
buf = nullptr;
}
} else {
buf_len = 128; /* Initial guess */
buf = static_cast<char*>(malloc(buf_len));
}
if (!buf) {
errno = ENOMEM;
return nullptr;
}
} while (((myerror = gethostname(buf, buf_len)) == 0 && !memchr(buf, '\0', buf_len)) || errno == ENAMETOOLONG);
if (myerror) { /* gethostname failed, abort. */
free(buf);
buf = nullptr;
}
}
return buf;
}
bool localhostip(std::string& hostname, std::string& ip)
{
char* hname = localhost();
if (hname == nullptr) {
return false;
}
hostname = hname;
struct hostent* hp = gethostbyname(hname);
if (hp == nullptr) {
return false;
}
while (hp->h_addr_list[0]) {
ip = inet_ntoa(*(struct in_addr*)*hp->h_addr_list++);
}
return true;
}
std::string dumphex(const std::string& str)
{
std::string hex;
dumphex(str.c_str(), str.size(), hex);
return hex;
}
void dumphex(const char data[], size_t size, std::string& hex)
{
hex.resize(size * 2);
for (size_t i = 0; i < size; i++) {
char c = data[i];
char h = (c >> 4) & 0x0F;
char l = c & 0x0F;
if (h >= 0x0A) {
h = h + 'A' - 10;
} else {
h = h + '0';
}
if (l >= 0x0A) {
l = l + 'A' - 10;
} else {
l = l + '0';
}
hex[i * 2] = h;
hex[i * 2 + 1] = l;
}
}
int hex2bin(const char data[], size_t size, std::string& bin)
{
if (size % 2 != 0) {
return -1;
}
bin.reserve(size / 2);
size_t count = 0;
char last_char = 0;
for (size_t i = 0; i < size; i++) {
char c = data[i];
if (c >= '0' && c <= '9') {
c -= '0';
} else if (c >= 'A' && c <= 'F') {
c = c - 'A' + 10;
} else if (c >= 'a' && c <= 'f') {
c = c - 'a' + 10;
} else {
continue;
}
if (count % 2 == 0) {
last_char = c << 4;
} else {
bin.append(1, (char)(last_char | c));
}
count++;
}
return 0;
}
} // namespace logproxy
} // namespace oceanbase

59
src/common/common.h Normal file
View File

@ -0,0 +1,59 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <string>
namespace oceanbase {
namespace logproxy {
#define OMS_OK 0
#define OMS_AGAIN 1
#define OMS_FAILED (-1)
#define OMS_TIMEOUT (-2)
#define OMS_CONNECT_FAILED (-3)
#define OMS_INVALID (-4)
#define OMS_UNUSED(x) (void)(x)
#define OMS_SINGLETON(__clazz__) \
public: \
static __clazz__& instance() \
{ \
static __clazz__ __clazz__##_singleton; \
return __clazz__##_singleton; \
} \
\
private: \
__clazz__() \
{}
#define OMS_AVOID_COPY(__clazz__) \
private: \
__clazz__(const __clazz__&) = delete; \
__clazz__& operator=(const __clazz__&) = delete;
#define OMS_ATOMIC_CAS(_it_, _old_, _new_) __sync_bool_compare_and_swap(&_it_, _old_, _new_)
#define OMS_ATOMIC_INC(_it_) __sync_add_and_fetch(&_it_, 1)
#define OMS_ATOMIC_DEC(_it_) __sync_add_and_fetch(&_it_, -1)
bool localhostip(std::string& hostname, std::string& ip);
std::string dumphex(const std::string& str);
void dumphex(const char data[], size_t size, std::string& hex);
int hex2bin(const char data[], size_t size, std::string& bin);
} // namespace logproxy
} // namespace oceanbase

61
src/common/config.cpp Normal file
View File

@ -0,0 +1,61 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <sstream>
#include <fstream>
#include "log.h"
#include "config.h"
#include "jsonutil.hpp"
namespace oceanbase {
namespace logproxy {
int Config::load(const std::string& file)
{
std::ifstream ifs(file, std::ios::in);
if (!ifs.good()) {
OMS_FATAL << "failed to open config file: " << file;
return OMS_FAILED;
}
std::string errmsg;
Json::Value json;
if (!str2json(ifs, json, &errmsg)) {
OMS_FATAL << "failed to parse config file: " << file << ", errmsg: " << errmsg;
return OMS_FAILED;
}
for (const std::string& k : json.getMemberNames()) {
const auto& centry = _configs.find(k);
if (_configs.count(k) != 0) {
centry->second->from_str(json[k].asString());
}
}
OMS_INFO << "success to load config: \n" << debug_str(true);
return OMS_OK;
}
std::string Config::debug_str(bool formatted) const
{
std::stringstream ss;
for (auto& entry : _configs) {
ss << entry.first << ":" << entry.second->debug_str() << ",";
if (formatted) {
ss << '\n';
}
}
return ss.str();
}
} // namespace logproxy
} // namespace oceanbase

122
src/common/config.h Normal file
View File

@ -0,0 +1,122 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <string>
#include <stdint.h>
#include "common/common.h"
#include "common/config_base.h"
namespace oceanbase {
namespace logproxy {
enum LogType {
/**
* OceanBase LogReader
*/
OCEANBASE = 0,
};
class Config : protected ConfigBase {
OMS_SINGLETON(Config);
OMS_AVOID_COPY(Config);
public:
virtual ~Config() = default;
int load(const std::string& file);
std::string debug_str(bool formatted = false) const;
public:
OMS_CONFIG_UINT16(service_port, 2983);
OMS_CONFIG_UINT32(encode_threadpool_size, 8);
OMS_CONFIG_UINT32(encode_queue_size, 20000);
OMS_CONFIG_UINT32(max_packet_bytes, 1024 * 1024 * 64); // 64MB
OMS_CONFIG_UINT32(command_timeout_s, 10);
OMS_CONFIG_UINT64(accept_interval_us, 500000);
OMS_CONFIG_UINT32(record_queue_size, 20000);
OMS_CONFIG_UINT64(read_timeout_us, 2000000);
OMS_CONFIG_UINT64(read_fail_interval_us, 1000000);
OMS_CONFIG_UINT32(read_wait_num, 20000);
OMS_CONFIG_UINT64(send_timeout_us, 2000000);
OMS_CONFIG_UINT64(send_fail_interval_us, 1000000);
OMS_CONFIG_BOOL(check_quota_enable, false);
OMS_CONFIG_BOOL(log_to_stdout, false);
OMS_CONFIG_UINT32(log_quota_size_mb, 5120);
OMS_CONFIG_UINT32(log_quota_day, 30);
OMS_CONFIG_UINT32(log_gc_interval_s, 43200);
OMS_CONFIG_STR(oblogreader_path, "./run");
OMS_CONFIG_UINT32(oblogreader_path_retain_hour, 168); // 7 Days
OMS_CONFIG_UINT32(oblogreader_lease_s, 300); // 5 mins
OMS_CONFIG_UINT32(oblogreader_max_count, 100);
OMS_CONFIG_UINT32(max_cpu_ratio, 0);
OMS_CONFIG_UINT64(max_mem_quota_mb, 0);
OMS_CONFIG_BOOL(allow_all_tenant, false);
OMS_CONFIG_BOOL(auth_user, true);
OMS_CONFIG_BOOL(auth_use_rs, false);
OMS_CONFIG_BOOL(auth_allow_sys_user, false);
OMS_CONFIG_ENCRYPT(ob_sys_username, "");
OMS_CONFIG_ENCRYPT(ob_sys_password, "");
OMS_CONFIG_UINT64(ob_clog_fetch_interval_s, 600); // 10 mins
OMS_CONFIG_UINT64(ob_clog_expr_s, 43200); // 12 h
OMS_CONFIG_UINT32(counter_interval_s, 2); // 2s
OMS_CONFIG_BOOL(metric_enable, true);
OMS_CONFIG_UINT32(metric_interval_s, 120); // 2mins
// when builtin_cluster_url_prefix not empty, we read cluster_id in handshake to make an complete cluster_url
OMS_CONFIG_STR(builtin_cluster_url_prefix, "");
OMS_CONFIG_STR(communication_mode, "server"); // server mode or client mode
// plain, tls refer ChannelFactory::init
OMS_CONFIG_STR(channel_type, "plain");
OMS_CONFIG_STR(tls_ca_cert_file, "");
OMS_CONFIG_STR(tls_cert_file, "");
OMS_CONFIG_STR(tls_key_file, "");
OMS_CONFIG_BOOL(tls_verify_peer, true);
// tls between observer and liboblog
OMS_CONFIG_BOOL(liboblog_tls, true);
OMS_CONFIG_STR(liboblog_tls_cert_path, "");
// debug related
OMS_CONFIG_BOOL(debug, false); // enable debug mode
OMS_CONFIG_BOOL(verbose, false); // print more log
OMS_CONFIG_BOOL(verbose_packet, false); // print data packet info
OMS_CONFIG_BOOL(readonly, false); // only read from LogReader, use for test
OMS_CONFIG_BOOL(count_record, false);
// for inner use
OMS_CONFIG_UINT64(process_name_address, 0);
OMS_CONFIG_BOOL(packet_magic, true);
OMS_CONFIG_UINT32(node_mem_limit_minimum_mb, 2048);
OMS_CONFIG_UINT32(node_mem_limit_threshold_percent, 85);
OMS_CONFIG_UINT32(node_cpu_limit_threshold_percent, 90);
OMS_CONFIG_UINT32(node_disk_limit_threshold_percent, 85);
};
} // namespace logproxy
} // namespace oceanbase

57
src/common/config_base.cpp Normal file
View File

@ -0,0 +1,57 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <sstream>
#include "config_base.h"
#include "log.h"
#include "ob_aes256.h"
namespace oceanbase {
namespace logproxy {
void ConfigBase::add_item(const std::string& key, ConfigItemBase* item)
{
_configs.emplace(key, item);
}
void EncryptedConfigItem::from_str(const std::string& val)
{
if (val.empty()) {
return;
}
const char* encrypt_key = nullptr;
if (!_encrypt_key.empty()) {
encrypt_key = _encrypt_key.c_str();
}
std::string bin_val;
hex2bin(val.data(), val.size(), bin_val);
char* decrypted = nullptr;
int decrypted_len = 0;
AES aes;
int ret = encrypt_key == nullptr
? aes.decrypt(bin_val.data(), bin_val.size(), &decrypted, decrypted_len)
: aes.decrypt(encrypt_key, bin_val.data(), bin_val.size(), &decrypted, decrypted_len);
if (ret != OMS_OK) {
OMS_ERROR << "Failed to decrypt: " << val;
exit(-1);
}
_val.assign(decrypted, decrypted_len);
free(decrypted);
}
} // namespace logproxy
} // namespace oceanbase

263
src/common/config_base.h Normal file
View File

@ -0,0 +1,263 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <stdint.h>
#include <stdlib.h>
#include <strings.h>
#include <string>
#include <map>
#include "common/common.h"
namespace oceanbase {
namespace logproxy {
class ConfigItemBase {
public:
virtual void from_str(const std::string& val) = 0;
virtual std::string debug_str() const = 0;
};
class ConfigBase {
public:
void add_item(const std::string& key, ConfigItemBase* item);
protected:
std::map<std::string, ConfigItemBase*> _configs;
};
template <typename T>
class ConfigItem : protected ConfigItemBase {
public:
friend class Config;
explicit ConfigItem(ConfigBase* config, const std::string& key, const T dft, bool optional = false)
: _key(key), _optional(optional)
{
_val = dft;
config->add_item(key, this);
}
const std::string& key() const
{
return _key;
}
void set(const T v)
{
_val = v;
}
T val() const
{
return _val;
}
bool empty() const
{
return empty_type(_val);
}
std::string debug_str() const override
{
return std::to_string(_val);
}
private:
void from_str(const std::string& val) override
{
_val = from_str(val, _val);
}
uint16_t from_str(const std::string& val, uint16_t v)
{
return atoi(val.c_str());
}
uint32_t from_str(const std::string& val, uint32_t v)
{
return atoi(val.c_str());
}
uint64_t from_str(const std::string& val, uint64_t v)
{
return strtoll(val.c_str(), nullptr, 10);
}
int16_t from_str(const std::string& val, int16_t v)
{
return atoi(val.c_str());
}
int32_t from_str(const std::string& val, int32_t v)
{
return atoi(val.c_str());
}
int64_t from_str(const std::string& val, int64_t v)
{
return strtoull(val.c_str(), nullptr, 10);
}
bool from_str(const std::string& val, bool v)
{
return strcasecmp("true", val.c_str()) == 0;
}
static bool empty_type(T v)
{
return false;
}
private:
std::string _key;
T _val;
bool _optional = false;
};
// specific for std::string
template <>
class ConfigItem<std::string> : protected ConfigItemBase {
public:
friend class Config;
explicit ConfigItem(ConfigBase* config, const std::string& key, const std::string& dft, bool optional = false)
: _key(key), _optional(optional)
{
_val = dft;
config->add_item(key, this);
}
const std::string& key() const
{
return _key;
}
void set(const std::string& v)
{
_val = v;
}
const std::string& val() const
{
return _val;
}
bool empty() const
{
return _val.empty();
}
std::string debug_str() const override
{
return _val;
}
protected:
void from_str(const std::string& val) override
{
_val = val;
}
protected:
std::string _key;
std::string _val;
bool _optional = false;
};
class EncryptedConfigItem : public ConfigItem<std::string> {
public:
friend class Config;
EncryptedConfigItem(ConfigBase* config, const std::string& key, const std::string& dft, bool optional = false,
const std::string& encrypt_key = "")
: ConfigItem<std::string>(config, key, dft, optional), _encrypt_key(encrypt_key)
{
_val = dft;
config->add_item(key, this);
}
std::string debug_str() const override
{
return "******";
}
private:
void from_str(const std::string& val) override;
private:
std::string _encrypt_key;
};
#define OMS_CONFIG_UINT16(key, args...) \
ConfigItem<uint16_t> key \
{ \
this, #key, ##args \
}
#define OMS_CONFIG_UINT32(key, args...) \
ConfigItem<uint32_t> key \
{ \
this, #key, ##args \
}
#define OMS_CONFIG_UINT64(key, args...) \
ConfigItem<uint64_t> key \
{ \
this, #key, ##args \
}
#define OMS_CONFIG_UINT64_K(name, key, args...) \
ConfigItem<uint64_t> name \
{ \
this, key, ##args \
}
#define OMS_CONFIG_INT16(key, args...) \
ConfigItem<int16_t> key \
{ \
this, #key, ##args \
}
#define OMS_CONFIG_INT32(key, args...) \
ConfigItem<int32_t> key \
{ \
this, #key, ##args \
}
#define OMS_CONFIG_INT64(key, args...) \
ConfigItem<int64_t> key \
{ \
this, #key, ##args \
}
#define OMS_CONFIG_BOOL(key, args...) \
ConfigItem<bool> key \
{ \
this, #key, ##args \
}
#define OMS_CONFIG_STR(key, args...) \
ConfigItem<std::string> key \
{ \
this, #key, ##args \
}
#define OMS_CONFIG_STR_K(name, key, args...) \
ConfigItem<std::string> name \
{ \
this, key, ##args \
}
#define OMS_CONFIG_ENCRYPT(key, args...) \
EncryptedConfigItem key \
{ \
this, #key, ##args \
}
} // namespace logproxy
} // namespace oceanbase

134
src/common/counter.cpp Normal file
View File

@ -0,0 +1,134 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <sstream>
#include "log.h"
#include "counter.h"
namespace oceanbase {
namespace logproxy {
void Counter::stop()
{
Thread::stop();
_sleep_cv.notify_all();
join();
}
void Counter::run()
{
OMS_INFO << "#### Counter thread running, tid: " << tid();
std::stringstream ss;
while (is_run()) {
_timer.reset();
this->sleep();
int64_t interval_ms = _timer.elapsed() / 1000;
int64_t interval_s = interval_ms == 0 ? 0 : (interval_ms / 1000);
uint64_t rcount = _read_count.load();
uint64_t wcount = _write_count.load();
uint64_t rio = _read_io.load();
uint64_t wio = _write_io.load();
uint64_t xwio = _xwrite_io.load();
uint64_t avg_size = wcount == 0 ? 0 : (wio / wcount);
uint64_t xavg_size = wcount == 0 ? 0 : (xwio / wcount);
uint64_t rrps = interval_s == 0 ? rcount : (rcount / interval_s);
uint64_t wrps = interval_s == 0 ? wcount : (wcount / interval_s);
uint64_t rios = interval_s == 0 ? rio : (rio / interval_s);
uint64_t wios = interval_s == 0 ? wio : (wio / interval_s);
uint64_t xwios = interval_s == 0 ? xwio : (xwio / interval_s);
int nowtm = time(nullptr);
int delay = nowtm - _timestamp;
int chk_delay = nowtm - _checkpoint;
// TODO... bytes rate
ss.str("");
ss << "Counter:[Span:" << interval_ms << "ms][Delay:" << delay << "," << chk_delay << "][RCNT:" << rcount
<< "][RRPS:" << rrps << "][RIOS:" << rios << "][WCNT:" << wcount << "][WRPS:" << wrps << "][WIOS:" << wios
<< ",AVG:" << avg_size << "][XWIOS:" << xwios << ",AVG:" << xavg_size << "]";
for (auto& count : _counts) {
uint64_t c = count.count.load();
ss << "[" << count.name << ":" << c << "]";
count.count.fetch_sub(c);
}
for (auto& entry : _gauges) {
ss << "[" << entry.first << ":" << entry.second() << "]";
}
OMS_INFO << ss.str();
// sub count that logged
_read_count.fetch_sub(rcount);
_write_count.fetch_sub(wcount);
_read_io.fetch_sub(rio);
_write_io.fetch_sub(wio);
_xwrite_io.fetch_sub(xwio);
}
OMS_INFO << "#### Counter thread stop, tid: " << tid();
}
void Counter::register_gauge(const std::string& key, const std::function<int64_t()>& func)
{
_gauges.emplace(key, func);
}
void Counter::count_read(int count)
{
_read_count.fetch_add(count);
}
void Counter::count_write(int count)
{
_write_count.fetch_add(count);
}
void Counter::count_read_io(int bytes)
{
_read_io.fetch_add(bytes);
}
void Counter::count_write_io(int bytes)
{
_write_io.fetch_add(bytes);
}
void Counter::count_xwrite_io(int bytes)
{
_xwrite_io.fetch_add(bytes);
}
void Counter::count_key(Counter::CountKey key, uint64_t count)
{
_counts[key].count.fetch_add(count);
}
void Counter::mark_timestamp(int timestamp)
{
_timestamp = timestamp;
}
void Counter::mark_checkpoint(uint64_t checkpoint)
{
_checkpoint = checkpoint;
}
void Counter::sleep()
{
// condition variable as SLEEP which could be gracefully interrupted
std::unique_lock<std::mutex> lk(_sleep_cv_lk);
_sleep_cv.wait_for(lk, std::chrono::seconds(_sleep_interval_s));
}
} // namespace logproxy
} // namespace oceanbase

99
src/common/counter.h Normal file
View File

@ -0,0 +1,99 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <time.h>
#include <unistd.h>
#include <atomic>
#include <mutex>
#include <condition_variable>
#include <map>
#include <functional>
#include "common/common.h"
#include "common/thread.h"
#include "common/timer.h"
#include "common/config.h"
namespace oceanbase {
namespace logproxy {
class Counter : public Thread {
OMS_SINGLETON(Counter);
OMS_AVOID_COPY(Counter);
public:
void stop() override;
void run() override;
void register_gauge(const std::string& key, const std::function<int64_t()>& func);
void count_read(int count = 1);
void count_write(int count = 1);
void count_read_io(int bytes);
void count_write_io(int bytes);
void count_xwrite_io(int bytes);
// MUST BE as same order as _counts
enum CountKey {
READER_FETCH_US = 0,
READER_OFFER_US = 1,
SENDER_POLL_US = 2,
SENDER_ENCODE_US = 3,
SENDER_SEND_US = 4,
};
void count_key(CountKey key, uint64_t count);
void mark_timestamp(int timestamp);
void mark_checkpoint(uint64_t checkpoint);
private:
void sleep();
private:
struct CountItem {
const char* name;
std::atomic<uint64_t> count{0};
CountItem(const char* n) : name(n)
{}
};
Timer _timer;
std::atomic<uint64_t> _read_count{0};
std::atomic<uint64_t> _write_count{0};
std::atomic<uint64_t> _read_io{0};
std::atomic<uint64_t> _write_io{0};
std::atomic<uint64_t> _xwrite_io{0};
volatile int _timestamp = time(nullptr);
volatile int _checkpoint = time(nullptr);
CountItem _counts[5]{{"RFETCH"}, {"ROFFER"}, {"SPOLL"}, {"SENCODE"}, {"SSEND"}};
std::map<std::string, std::function<int64_t()>> _gauges;
std::mutex _sleep_cv_lk;
std::condition_variable _sleep_cv;
uint64_t _sleep_interval_s = Config::instance().counter_interval_s.val();
};
} // namespace logproxy
} // namespace oceanbase

110
src/common/file_gc.cpp Normal file
View File

@ -0,0 +1,110 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <dirent.h>
#include <unistd.h>
#include <sys/stat.h>
#include <string.h>
#include "common/config.h"
#include "common/log.h"
#include "common/file_gc.h"
namespace oceanbase {
namespace logproxy {
static Config& _s_config = Config::instance();
FileGcRoutine::FileGcRoutine(std::string path, std::set<std::string> prefixs)
{
_path = std::move(path);
_prefixs = std::move(prefixs);
}
void FileGcRoutine::run()
{
OMS_INFO << "file gc size quota MB: " << _s_config.log_quota_size_mb.val() * 1024;
OMS_INFO << "file gc time quota day: " << _s_config.log_quota_day.val();
OMS_INFO << "file gc path: " << _path;
for (auto& prefix : _prefixs) {
OMS_INFO << "file gc prefixs: " << prefix;
}
while (is_run()) {
DIR* dir = ::opendir(_path.c_str());
if (dir == nullptr) {
sleep(_s_config.log_gc_interval_s.val());
continue;
}
// <mode time, <file, file size in KB, is_del>>
std::map<uint32_t, std::tuple<std::string, uint64_t, bool>> todels;
time_t nowtime = time(nullptr);
struct dirent* dent = nullptr;
while ((dent = readdir(dir)) != nullptr) {
for (auto& prefix : _prefixs) {
if (dent->d_type != DT_REG) {
continue;
}
if (strncmp(dent->d_name, prefix.c_str(), prefix.size()) != 0) {
continue;
}
std::string filename = _path + "/" + dent->d_name;
struct stat st;
if (::stat(filename.c_str(), &st) != 0) {
OMS_INFO << "file to gc stat failed: " << filename;
continue;
}
bool isdel = false;
if (nowtime - st.st_mtime >= (_s_config.log_quota_day.val() * 86400)) {
isdel = true;
}
todels.emplace((uint32_t)st.st_mtime, std::make_tuple(filename, (uint64_t)st.st_size / 1024, isdel));
}
}
closedir(dir);
// from new to old
uint64_t size_count = 0;
for (auto iter = todels.rbegin(); iter != todels.rend(); ++iter) {
if (std::get<2>(iter->second)) {
continue;
}
size_count += std::get<1>(iter->second);
if (size_count >= (_s_config.log_quota_size_mb.val() * 1024LL)) {
std::get<2>(iter->second) = true;
// rest of files all will be marked to delete as size_count are still accumulating
}
}
for (auto& todel : todels) {
if (std::get<2>(todel.second)) {
OMS_INFO << "file gc: " << std::get<0>(todel.second);
int ret = remove(std::get<0>(todel.second).c_str());
if (ret != 0) {
OMS_WARN << "Failed to remove: " << std::get<0>(todel.second) << ". error=" << strerror(errno);
}
}
}
sleep(_s_config.log_gc_interval_s.val());
}
}
} // namespace logproxy
} // namespace oceanbase

36
src/common/file_gc.h Normal file
View File

@ -0,0 +1,36 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <string>
#include <set>
#include "common/thread.h"
namespace oceanbase {
namespace logproxy {
class FileGcRoutine : public Thread {
public:
FileGcRoutine(std::string path, std::set<std::string> prefixs);
protected:
void run() override;
private:
std::string _path;
std::set<std::string> _prefixs;
};
} // namespace logproxy
} // namespace oceanbase

222
src/common/fs_util.cpp Normal file
View File

@ -0,0 +1,222 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <libgen.h>
#include <unistd.h>
#include "openssl/md5.h"
#include "common/log.h"
#include "common/fs_util.h"
#include "common/common.h"
#include "str.h"
namespace oceanbase {
namespace logproxy {
bool FsUtil::exist(const std::string& path)
{
struct stat info;
int ret = stat(path.c_str(), &info);
if (ret == 0) {
return true;
}
if (errno == ENOENT) {
return false;
}
OMS_ERROR << "Can't access the path: " << path;
return false;
}
bool FsUtil::mkdir(const std::string& path)
{
if (path.empty() || exist(path)) {
return true;
}
// TODO... recursive
if (::mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) != 0) {
OMS_ERROR << "Failed to mkdir, path: " << path << ". system error:" << strerror(errno);
return false;
}
return true;
}
bool FsUtil::remove(const std::string& dest)
{
if (!remove_subs(dest)) {
return false;
}
int ret = ::remove(dest.c_str());
if (ret != 0) {
OMS_ERROR << "Failed to remove " << dest << ", ret: " << ret << ", errno: " << errno
<< ", error: " << strerror(errno);
return false;
}
return true;
}
bool FsUtil::remove_subs(const std::string& dest)
{
if (!exist(dest)) {
return true;
}
DIR* cur_dir = opendir(dest.c_str());
if (cur_dir == nullptr) {
OMS_ERROR << "Failed to opendir: " << dest;
return false;
}
struct dirent* ent = nullptr;
while ((ent = readdir(cur_dir)) != nullptr) {
if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) {
continue;
}
if (!remove(dest + "/" + ent->d_name)) {
OMS_ERROR << "failed to remove: " << dest;
closedir(cur_dir);
return false;
}
}
closedir(cur_dir);
return true;
}
bool FsUtil::write_file(const std::string& filename, const std::string& content)
{
FILE* fp = ::fopen(filename.c_str(), "w+");
if (fp == nullptr) {
OMS_ERROR << "Failed to open " << filename << ", errno: " << errno << ", error: " << strerror(errno);
return false;
}
size_t n = fwrite(content.c_str(), content.size(), 1, fp);
fclose(fp);
if (n != 1) {
OMS_ERROR << "Failed to write file, length error, file_path: " << filename;
return false;
}
return true;
}
void FsUtil::dir_foreach(const std::string& dir, const std::function<void(const std::string&, const FileType)>& func)
{
DIR* cur_dir = opendir(dir.c_str());
if (cur_dir == nullptr) {
OMS_ERROR << "Failed to opendir " << dir;
return;
}
struct dirent* ent = nullptr;
while ((ent = readdir(cur_dir)) != nullptr) {
if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) {
continue;
}
if (ent->d_type == DT_REG || ent->d_type == DT_DIR) {
func(ent->d_name, (FileType)ent->d_type);
}
}
closedir(cur_dir);
}
std::string FsUtil::file_md5(const std::string& filename)
{
MD5_CTX ctx;
MD5_Init(&ctx);
FILE* fp = fopen(filename.c_str(), "rb");
if (fp == nullptr) {
return "";
}
unsigned char buf[8092];
size_t count = 0;
while ((count = fread(buf, 1, sizeof(buf), fp)) > 0) {
MD5_Update(&ctx, buf, count);
}
fclose(fp);
buf[0] = '\0';
MD5_Final(buf, &ctx);
size_t idx = 0;
char md5[33] = "\0";
for (size_t i = 0; i < 16; ++i) {
idx += snprintf(md5 + idx, 32, "%02x", buf[i]);
}
return {md5};
}
bool FsUtil::read_file(const std::string& filename, std::string& content, bool trim_content)
{
std::ifstream ifs(filename);
if (!ifs.good()) {
OMS_ERROR << "Failed to open: " << filename << " errno:" << errno;
return false;
}
std::stringstream buffer;
buffer << ifs.rdbuf();
content = buffer.str();
if (trim_content) {
trim(content);
}
return true;
}
bool FsUtil::read_number(const std::string& filename, int64_t& number)
{
std::string val;
if (!read_file(filename, val)) {
return false;
}
number = strtoll(val.c_str(), nullptr, 10);
return true;
}
bool FsUtil::read_lines(const std::string& filename, std::vector<std::string>& lines)
{
std::ifstream ifs(filename);
if (!ifs.good()) {
OMS_ERROR << "Failed to open: " << filename << " errno:" << errno;
return false;
}
for (std::string line; std::getline(ifs, line);) {
lines.emplace_back(std::move(line));
}
return true;
}
bool FsUtil::read_kvs(const std::string& filename, const std::string& sep, std::map<std::string, std::string>& kvs)
{
if (sep.empty()) {
OMS_ERROR << "Invalid empty seperator";
return false;
}
std::vector<std::string> lines;
if (!read_lines(filename, lines)) {
return false;
}
for (const std::string& line : lines) {
std::vector<std::string> kv(2);
split_by_str(line, sep, kv);
if (kv.size() != 2) {
continue;
}
kvs.emplace(std::move(kv[0]), std::move(kv[1]));
}
return true;
}
} // namespace logproxy
} // namespace oceanbase

52
src/common/fs_util.h Normal file
View File

@ -0,0 +1,52 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <string>
#include <functional>
#include <dirent.h>
#include <sys/stat.h>
#include <fstream>
namespace oceanbase {
namespace logproxy {
class FsUtil {
public:
typedef enum { FS_DIR = DT_DIR, FS_FILE = DT_REG } FileType;
static bool exist(const std::string& path);
static bool mkdir(const std::string& dest);
static bool remove(const std::string& dest);
static bool remove_subs(const std::string& dest);
static bool write_file(const std::string& filename, const std::string& content);
static void dir_foreach(const std::string& dir, const std::function<void(const std::string&, const FileType)>& func);
static std::string file_md5(const std::string& filename);
static bool read_file(const std::string& filename, std::string& content, bool trim_content = true);
static bool read_number(const std::string& filename, int64_t& number);
static bool read_lines(const std::string& filename, std::vector<std::string>& lines);
static bool read_kvs(const std::string& filename, const std::string& sep, std::map<std::string, std::string>& kvs);
};
} // namespace logproxy
} // namespace oceanbase

55
src/common/guard.hpp Normal file
View File

@ -0,0 +1,55 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <stdlib.h>
#include <functional>
namespace oceanbase {
namespace logproxy {
template <typename T>
class FreeGuard {
public:
explicit FreeGuard(const T ptr) : _ptr(ptr)
{}
FreeGuard(T ptr, std::function<void(T)> free_func) : _ptr(ptr), _free_func(free_func)
{}
~FreeGuard()
{
if (_own && _ptr != nullptr) {
if (_free_func) {
_free_func(_ptr);
} else {
::free(_ptr);
}
_ptr = nullptr;
}
}
void release()
{
_own = false;
}
private:
bool _own = true;
T _ptr = nullptr;
std::function<void(T)> _free_func;
};
} // namespace logproxy
} // namespace oceanbase

17
src/common/jsonutil.cpp Normal file
View File

@ -0,0 +1,17 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "jsonutil.hpp"
namespace oceanbase {
namespace logproxy {}
} // namespace oceanbase

48
src/common/jsonutil.hpp Normal file
View File

@ -0,0 +1,48 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include "json/json.h"
namespace oceanbase {
namespace logproxy {
template <class JT>
std::string json2str(const JT& json, bool formate = false)
{
Json::StreamWriterBuilder builder;
if (!formate) {
builder.settings_["indentation"] = "";
}
return Json::writeString(builder, json);
}
template <class JT>
bool str2json(const std::string& str, JT& json, std::string* errmsg = nullptr)
{
Json::CharReaderBuilder builder;
builder["collectComments"] = false;
std::unique_ptr<Json::CharReader> reader(builder.newCharReader());
return reader->parse(str.data(), str.data() + str.size(), &json, errmsg);
}
template <class JT>
bool str2json(std::ifstream& ifs, JT& json, std::string* errmsg = nullptr)
{
Json::CharReaderBuilder builder;
builder["collectComments"] = false;
return Json::parseFromStream(builder, (Json::IStream&)ifs, &json, errmsg);
}
} // namespace logproxy
} // namespace oceanbase

73
src/common/log.cpp Normal file
View File

@ -0,0 +1,73 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "common/log.h"
#include "common/config.h"
namespace oceanbase {
namespace logproxy {
// static char dump_line[1024];
// static void dump_writer(const char* buf, int size)
//{
// int len = std::min(1022, size);
// memcpy(dump_line, buf, len);
// dump_line[len] = '\n';
// dump_line[len + 1] = '\0';
// printf(dump_line);
// }
void init_log(const char* argv0, bool restart)
{
#ifdef WITH_GLOG
if (Config::instance().log_to_stdout.val()) {
return;
}
if (restart) {
google::ShutdownGoogleLogging();
}
std::string bin_name = argv0;
auto pos = bin_name.find_last_of('/');
if (pos != std::string::npos) {
bin_name = bin_name.substr(pos + 1);
}
FLAGS_minloglevel = 0;
FLAGS_logbufsecs = 0;
FLAGS_max_log_size = 1024;
FLAGS_stop_logging_if_full_disk = true;
FLAGS_logtostderr = false;
FLAGS_alsologtostderr = false;
FLAGS_log_dir = "";
google::SetLogSymlink(google::GLOG_INFO, bin_name.c_str());
google::SetLogSymlink(google::GLOG_WARNING, bin_name.c_str());
google::SetLogSymlink(google::GLOG_ERROR, bin_name.c_str());
google::SetLogSymlink(google::GLOG_FATAL, bin_name.c_str());
google::SetLogDestination(google::GLOG_INFO, "log/logproxy_info.");
google::SetLogDestination(google::GLOG_WARNING, "log/logproxy_warn.");
google::SetLogDestination(google::GLOG_ERROR, "log/logproxy_error.");
google::SetLogDestination(google::GLOG_FATAL, "log/logproxy_error.");
// !!!NOTICE!!! Nerver call glog InstallFailureSignalHandler() causing corrupt liboblog coredump-signal-recover
// features
google::InitGoogleLogging(bin_name.c_str());
#endif
}
} // namespace logproxy
} // namespace oceanbase

323
src/common/log.h Normal file
View File

@ -0,0 +1,323 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <vector>
#include <deque>
#include <map>
#include <unordered_map>
#include <set>
#include <unordered_set>
#include <sstream>
#include <iostream>
#if __cplusplus >= 201703L
#include <list>
#endif
#include <sys/time.h>
namespace oceanbase {
namespace logproxy {
class LogStream {
public:
explicit LogStream(int level, const std::string& file, const int line, std::ostream* os = &std::cout)
: _level(level), _file(file), _line(line), _os(os)
{
char tmbuf[2 + 2 + 1 + 2 + 1 + 2 + 1 + 2 + 1 + 6 + 1 + 1 + 5 + 1 + 1];
if (_level == 0) {
#ifndef NDEBUG
prefix(tmbuf, sizeof(tmbuf));
_pre_str = "D" + std::string(tmbuf) + ")[" + file + ":" + std::to_string(line) + "] ";
#endif
return;
}
prefix(tmbuf, sizeof(tmbuf));
if (_level == 1) {
_pre_str = "I" + std::string(tmbuf) + ")[" + file + ":" + std::to_string(line) + "] ";
} else if (_level == 2) {
_pre_str = "W" + std::string(tmbuf) + ")[" + file + ":" + std::to_string(line) + "] ";
} else if (_level == 3) {
_pre_str = "E" + std::string(tmbuf) + ")[" + file + ":" + std::to_string(line) + "] ";
} else if (_level == 4) {
_pre_str = "F" + std::string(tmbuf) + ")[" + file + ":" + std::to_string(line) + "] ";
}
}
virtual ~LogStream()
{
flush();
}
void flush()
{
if (_level == 0) {
#ifndef NDEBUG
if (_os != nullptr) {
*_os << _pre_str << _ss.str() << std::endl;
}
_ss.clear();
#endif
return;
}
if (_os != nullptr) {
*_os << _pre_str << _ss.str() << std::endl;
}
_ss.clear();
}
std::string str(bool prefix = false)
{
return prefix ? (_pre_str + _ss.str()) : _ss.str();
}
inline void prefix(char* pre_buf, size_t len)
{
struct timeval tv;
::gettimeofday(&tv, nullptr);
struct tm ltm;
::localtime_r(&tv.tv_sec, &ltm);
// tuncate tailing pthread id
snprintf(pre_buf,
len,
"%02d%02d %02d:%02d:%02d.%06ld (%5lx",
ltm.tm_mon + 1,
ltm.tm_mday,
ltm.tm_hour,
ltm.tm_min,
ltm.tm_sec,
tv.tv_usec,
(uint64_t)pthread_self());
}
inline LogStream& operator<<(int16_t val)
{
_ss << val;
return *this;
}
inline LogStream& operator<<(int val)
{
_ss << val;
return *this;
}
inline LogStream& operator<<(int64_t val)
{
_ss << val;
return *this;
}
inline LogStream& operator<<(uint16_t val)
{
_ss << val;
return *this;
}
inline LogStream& operator<<(uint32_t val)
{
_ss << val;
return *this;
}
inline LogStream& operator<<(uint64_t val)
{
_ss << val;
return *this;
}
inline LogStream& operator<<(float val)
{
_ss << val;
return *this;
}
#ifdef __APPLE__
inline LogStream& operator<<(size_t val)
{
_ss << val;
return *this;
}
inline LogStream& operator<<(ssize_t val)
{
_ss << val;
return *this;
}
inline LogStream& operator<<(pthread_t tid)
{
_ss << (uint64_t)tid;
return *this;
}
#endif
inline LogStream& operator<<(double val)
{
_ss << val;
return *this;
}
inline LogStream& operator<<(const std::string& val)
{
_ss << val;
return *this;
}
template <typename T>
inline LogStream& operator<<(const std::vector<T>& t)
{
iter_list(t);
return *this;
}
template <typename T>
inline LogStream& operator<<(const std::deque<T>& t)
{
iter_list(t);
return *this;
}
#if __cplusplus >= 201703L
template <typename T>
inline LogStream& operator<<(const std::list<T>& t)
{
iter_list(t);
return *this;
}
#endif
template <typename T>
inline LogStream& operator<<(const std::set<T>& t)
{
iter_list(t);
return *this;
}
template <typename T>
inline LogStream& operator<<(const std::unordered_set<T>& t)
{
iter_list(t);
return *this;
}
template <typename K, typename V>
inline LogStream& operator<<(const std::map<K, V>& t)
{
iter_kv(t);
return *this;
}
template <typename K, typename V>
inline LogStream& operator<<(const std::unordered_map<K, V>& t)
{
iter_kv(t);
return *this;
}
template <typename K, typename V>
inline LogStream& operator<<(const std::pair<K, V>& t)
{
*this << t.first << "," << t.second;
return *this;
}
template <typename T>
void iter_list(const T& t)
{
*this << "[";
size_t i = 0;
for (auto iter = t.begin(); iter != t.end(); ++iter) {
*this << *iter;
if (i++ != t.size() - 1) {
*this << ",";
}
}
*this << "]";
}
template <typename T>
void iter_kv(const T& t)
{
*this << "{";
size_t i = 0;
for (auto iter = t.begin(); iter != t.end(); ++iter) {
*this << iter->first << ":" << iter->second;
if (i++ != t.size() - 1) {
*this << ",";
}
}
*this << "}";
}
private:
std::stringstream _ss;
int _level;
const std::string& _file;
const int _line;
std::string _pre_str;
std::ostream* _os = nullptr;
};
template <typename T>
class LogMessage {
public:
explicit LogMessage(int level, const std::string& file, const int line)
{
if (_stream == nullptr) {
_stream = new T(level, file, line);
}
}
~LogMessage()
{
if (_stream != nullptr) {
_stream->flush();
}
}
T& stream()
{
return *_stream;
}
private:
T* _stream = nullptr;
};
void init_log(const char* argv0, bool restart = false);
} // namespace logproxy
} // namespace oceanbase
#ifdef WITH_GLOG
#include "glog/logging.h"
#define OMS_DEBUG DLOG(INFO)
#define OMS_INFO LOG(INFO)
#define OMS_WARN LOG(WARNING)
#define OMS_ERROR LOG(ERROR)
#define OMS_FATAL LOG(ERROR)
#else
#define OMS_DEBUG oceanbase::logproxy::LogMessage<oceanbase::logproxy::LogStream>(0, __FILE__, __LINE__).stream()
#define OMS_INFO oceanbase::logproxy::LogMessage<oceanbase::logproxy::LogStream>(1, __FILE__, __LINE__).stream()
#define OMS_WARN oceanbase::logproxy::LogMessage<oceanbase::logproxy::LogStream>(2, __FILE__, __LINE__).stream()
#define OMS_ERROR oceanbase::logproxy::LogMessage<oceanbase::logproxy::LogStream>(3, __FILE__, __LINE__).stream()
#define OMS_FATAL oceanbase::logproxy::LogMessage<oceanbase::logproxy::LogStream>(4, __FILE__, __LINE__).stream()
#endif

49
src/common/md5.cpp Normal file
View File

@ -0,0 +1,49 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <stdio.h>
#include "common/md5.h"
namespace oceanbase {
namespace logproxy {
Md5::Md5()
{
MD5_Init(&_ctx);
}
Md5::Md5(const char* data, size_t size)
{
MD5_Init(&_ctx);
update(data, size);
}
void Md5::update(const char* data, size_t size)
{
MD5_Update(&_ctx, data, size);
}
std::string Md5::done()
{
unsigned char buf[MD5_DIGEST_LENGTH] = "\0";
MD5_Final(buf, &_ctx);
size_t idx = 0;
char md5[33] = "\0";
for (unsigned char i : buf) {
idx += snprintf(md5 + idx, 32, "%02x", i);
}
return {md5};
}
} // namespace logproxy
} // namespace oceanbase

36
src/common/md5.h Normal file
View File

@ -0,0 +1,36 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <string>
#include "openssl/md5.h"
namespace oceanbase {
namespace logproxy {
class Md5 {
public:
Md5();
Md5(const char* data, size_t size);
void update(const char* data, size_t size);
std::string done();
private:
MD5_CTX _ctx;
};
} // namespace logproxy
} // namespace oceanbase

109
src/common/model.h Normal file
View File

@ -0,0 +1,109 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <vector>
#include <functional>
#include "common/common.h"
#include "common/log.h"
namespace oceanbase {
namespace logproxy {
class Model {
public:
Model() = default;
// copy ctor do nothing
Model(const Model& rhs)
{}
Model(Model&& rhs) noexcept
{}
Model& operator=(const Model& rhs)
{
_to_str_flag = false;
_to_str_buf.clear();
return *this;
}
Model& operator=(Model&& rhs) noexcept
{
_to_str_flag = false;
_to_str_buf.clear();
return *this;
}
const std::string& to_string() const
{
Model* p = (Model*)this;
if (OMS_ATOMIC_CAS(p->_to_str_flag, false, true)) {
LogStream ls{0, "", 0, nullptr};
for (auto& fn : p->_to_str_fns) {
fn(ls);
}
p->_to_str_buf = ls.str();
}
return _to_str_buf;
}
protected:
template <typename T>
struct Reg {
public:
Reg(Model* model, const std::string& k, T& v)
{
model->_to_str_fns.push_back([k, &v](LogStream& ls) { ls << k << ":" << v << ", "; });
}
};
volatile bool _to_str_flag = false;
std::vector<std::function<void(LogStream&)>> _to_str_fns;
std::string _to_str_buf;
};
#define OMS_MF(T, obj) OMS_MF_SCOPE(T, obj, public)
#define OMS_MF_PRI(T, obj) OMS_MF_SCOPE(T, obj, private)
#define OMS_MF_SCOPE(T, obj, scope) \
scope: \
T obj; \
\
private: \
Reg<T> _reg_##obj \
{ \
this, #obj, obj \
}
#define OMS_MF_DFT(T, obj, dft) OMS_MF_DFT_SCOPE(T, obj, dft, public)
#define OMS_MF_DFT_PRI(T, obj, dft) OMS_MF_DFT_SCOPE(T, obj, dft, private)
#define OMS_MF_DFT_SCOPE(T, obj, dft, scope) \
scope: \
T obj = dft; \
\
private: \
Reg<T> _reg_##obj \
{ \
this, #obj, obj \
}
#define OMS_MF_ENABLE_COPY(clazz) \
public: \
clazz(const clazz& rhs) : Model(rhs) \
{ \
*this = rhs; \
} \
clazz(clazz&& rhs) noexcept = default; \
clazz& operator=(const clazz& rhs) = default; \
clazz& operator=(clazz&& rhs) = default
} // namespace logproxy
} // namespace oceanbase

129
src/common/ob_aes256.cpp Normal file
View File

@ -0,0 +1,129 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "common/ob_aes256.h"
#include "common/common.h"
#include "common/log.h"
namespace oceanbase {
namespace logproxy {
static const char* default_encrypt_key = "LogProxy*";
static const unsigned char iv[] = "0123456789012345";
AES::AES()
{
_cipher_ctx = EVP_CIPHER_CTX_new();
}
AES::~AES()
{
EVP_CIPHER_CTX_free(_cipher_ctx);
}
int AES::reset()
{
EVP_CIPHER_CTX_cleanup(_cipher_ctx);
EVP_CIPHER_CTX_init(_cipher_ctx);
return OMS_OK;
}
int AES::encrypt(const char* plain_text, int plain_text_len, char** encrypted, int& encrypted_len)
{
return encrypt(default_encrypt_key, plain_text, plain_text_len, encrypted, encrypted_len);
}
int AES::encrypt(const char* key, const char* plain_text, int plain_text_len, char** encrypted, int& encrypted_len)
{
unsigned char iv_copy[sizeof(iv)];
memcpy(iv_copy, iv, sizeof(iv));
int ret = EVP_EncryptInit_ex(_cipher_ctx, EVP_aes_256_cbc(), nullptr, (const unsigned char*)key, iv_copy);
if (ret != 1) {
OMS_ERROR << "Failed to init encrypt, return=" << ret;
return OMS_FAILED;
}
const int buffer_len = plain_text_len + EVP_MAX_BLOCK_LENGTH;
unsigned char* buffer = (unsigned char*)malloc(buffer_len);
if (nullptr == buffer) {
OMS_ERROR << "Failed to alloc memory. size=" << buffer_len;
return OMS_FAILED;
}
int used_buffer_len = 0;
ret = EVP_EncryptUpdate(_cipher_ctx, buffer, &used_buffer_len, (const unsigned char*)plain_text, plain_text_len);
if (ret != 1) {
OMS_ERROR << "Failed to encrypt. returned=" << ret;
free(buffer);
return OMS_FAILED;
}
int final_buffer_len = 0;
ret = EVP_EncryptFinal_ex(_cipher_ctx, buffer + used_buffer_len, &final_buffer_len);
if (ret != 1) {
OMS_ERROR << "Failed to encrypt(final). returned=" << ret;
free(buffer);
return OMS_FAILED;
}
*encrypted = (char*)buffer;
encrypted_len = used_buffer_len + final_buffer_len;
return OMS_OK;
}
int AES::decrypt(const char* encrypted, int encrypted_len, char** plain_text, int& plain_text_len)
{
return decrypt(default_encrypt_key, encrypted, encrypted_len, plain_text, plain_text_len);
}
int AES::decrypt(const char* key, const char* encrypted, int encrypted_len, char** plain_text, int& plain_text_len)
{
unsigned char iv_copy[sizeof(iv)];
memcpy(iv_copy, iv, sizeof(iv));
int ret = EVP_DecryptInit_ex(_cipher_ctx, EVP_aes_256_cbc(), nullptr, (const unsigned char*)key, iv_copy);
if (ret != 1) {
OMS_ERROR << "Failed to init encrypt, return=" << ret;
return OMS_FAILED;
}
unsigned char* buffer = (unsigned char*)malloc(encrypted_len);
if (nullptr == buffer) {
OMS_ERROR << "Failed to alloc memory. size=" << encrypted_len;
return OMS_FAILED;
}
memset(buffer, 0, encrypted_len);
int used_buffer_len = 0;
ret = EVP_DecryptUpdate(_cipher_ctx, buffer, &used_buffer_len, (const unsigned char*)encrypted, encrypted_len);
if (ret != 1) {
OMS_ERROR << "Failed to decrypt. return=" << ret;
free(buffer);
return OMS_FAILED;
}
int final_buffer_len = 0;
ret = EVP_DecryptFinal_ex(_cipher_ctx, buffer + used_buffer_len, &final_buffer_len);
if (ret != 1) {
OMS_ERROR << "Failed to decrypt(final), ret:" << ret;
free(buffer);
return OMS_FAILED;
}
*plain_text = (char*)buffer;
plain_text_len = used_buffer_len + final_buffer_len;
return OMS_OK;
}
} // namespace logproxy
} // namespace oceanbase

37
src/common/ob_aes256.h Normal file
View File

@ -0,0 +1,37 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <openssl/aes.h>
#include <openssl/evp.h>
namespace oceanbase {
namespace logproxy {
class AES {
public:
AES();
~AES();
int reset();
int encrypt(const char* plain_text, int plain_text_len, char** encrypted, int& encrypted_len);
int encrypt(const char* key, const char* plain_text, int plain_text_len, char** encrypted, int& encrypted_len);
int decrypt(const char* encrypted, int encrypted_len, char** plain_text, int& plain_text_len);
int decrypt(const char* key, const char* encrypted, int encrypted_len, char** plain_text, int& plain_text_len);
private:
EVP_CIPHER_CTX* _cipher_ctx;
};
} // namespace logproxy
} // namespace oceanbase

84
src/common/option.cpp Normal file
View File

@ -0,0 +1,84 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "option.h"
namespace oceanbase {
namespace logproxy {
OmsOption::OmsOption(char opt, const std::string& opt_long, bool is_arg, const std::string& desc,
const std::function<void(const std::string&)>& func)
: opt(opt), opt_long(opt_long), is_arg(is_arg), desc(desc), func(func)
{}
OmsOptions::OmsOptions(const std::string& title) : _title(title)
{}
OmsOptions::~OmsOptions()
{
delete[] _sys_opts;
_sys_opts = nullptr;
}
void OmsOptions::add(const OmsOption& option)
{
_options.emplace(option.opt, option);
}
void OmsOptions::usage() const
{
printf("%s usage: \n", _title.c_str());
for (auto& opt : _options) {
printf("-%c, --%s\t\t\t%s\n", opt.first, opt.second.opt_long.c_str(), opt.second.desc.c_str());
}
}
struct option* OmsOptions::long_pattern()
{
if (_sys_opts == nullptr) {
_sys_opts = new option[_options.size() + 1];
_sys_opts[_options.size()] = {nullptr, 0, nullptr, 0};
}
size_t i = 0;
for (auto& entry : _options) {
_sys_opts[i].name = entry.second.opt_long.c_str();
_sys_opts[i].has_arg = entry.second.is_arg ? required_argument : no_argument;
_sys_opts[i].flag = nullptr;
_sys_opts[i].val = entry.second.opt;
++i;
}
return _sys_opts;
}
std::string OmsOptions::pattern()
{
std::string pt;
for (auto& entry : _options) {
pt.append(1, entry.second.opt);
if (entry.second.is_arg) {
pt.append(":");
}
}
return pt;
}
OmsOption* OmsOptions::get(char opt)
{
const auto& entry = _options.find(opt);
if (entry == _options.end()) {
return nullptr;
}
return &entry->second;
}
} // namespace logproxy
} // namespace oceanbase

67
src/common/option.h Normal file
View File

@ -0,0 +1,67 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <string>
#include <functional>
#include <unordered_map>
#include <getopt.h>
#include "common/common.h"
namespace oceanbase {
namespace logproxy {
struct OmsOption {
char opt;
std::string opt_long;
bool is_arg;
std::string desc;
std::function<void(const std::string&)> func;
OmsOption(char opt, const std::string& opt_long, bool is_arg, const std::string& desc,
const std::function<void(const std::string&)>& func);
};
class OmsOptions {
OMS_AVOID_COPY(OmsOptions);
public:
explicit OmsOptions(const std::string& title = "");
virtual ~OmsOptions();
void add(const OmsOption& option);
void usage() const;
struct option* long_pattern();
std::string pattern();
OmsOption* get(char opt);
inline std::unordered_map<char, OmsOption>& options()
{
return _options;
}
private:
std::string _title;
std::unordered_map<char, OmsOption> _options;
struct option* _sys_opts = nullptr;
};
} // namespace logproxy
} // namespace oceanbase

38
src/common/shell_executor.cpp Normal file
View File

@ -0,0 +1,38 @@
// Copyright (c) 2021 OceanBase
// OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
// You can use this software according to the terms and conditions of the Mulan PubL v2.
// You may obtain a copy of Mulan PubL v2 at:
// http://license.coscl.org.cn/MulanPubL-2.0
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
// EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
// MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
// See the Mulan PubL v2 for more details.
#include "common/shell_executor.h"
namespace oceanbase {
namespace logproxy {
int exec_cmd(const std::string& cmd, std::string& result)
{
FILE* fd = popen(cmd.c_str(), "r");
if (fd == nullptr) {
OMS_ERROR << "Failed to exec command:" << cmd << ", errno:" << errno << ", error:" << strerror(errno);
return -1;
}
char buff[1024];
result.clear();
while (fgets(buff, sizeof(buff), fd) != nullptr) {
buff[1023] = '\0';
result += buff;
}
int ret = pclose(fd);
ret = WEXITSTATUS(ret);
// OMS_DEBUG << "bash result code:" << ret << "(" << strerror(errno) << ")"
// << ", content: " << result;
// we set SIGCHILD to SIG_IGN cause pclose got an ECHILD errno
return (ret == 255 && errno == ECHILD) ? 0 : ret;
}
} // namespace logproxy
} // namespace oceanbase

21
src/common/shell_executor.h Normal file
View File

@ -0,0 +1,21 @@
// Copyright (c) 2021 OceanBase
// OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
// You can use this software according to the terms and conditions of the Mulan PubL v2.
// You may obtain a copy of Mulan PubL v2 at:
// http://license.coscl.org.cn/MulanPubL-2.0
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
// EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
// MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
// See the Mulan PubL v2 for more details.
#include <string>
#include <array>
#include <memory>
#include "common/log.h"
namespace oceanbase {
namespace logproxy {
int exec_cmd(const std::string& cmd, std::string& result);
} // namespace logproxy
} // namespace oceanbase

99
src/common/str.cpp Normal file
View File

@ -0,0 +1,99 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <algorithm>
#include "common/str.h"
namespace oceanbase {
namespace logproxy {
size_t split_all(const std::string& str, const std::string& seps, std::vector<std::string>& vec, bool first)
{
bool sep_idx[256] = {false};
for (char sep : seps) {
sep_idx[(uint8_t)sep] = true;
}
vec.clear();
if (str.empty()) {
return 0;
}
bool flag = false;
char* p_cur = const_cast<char*>(str.c_str());
char* p_pre = const_cast<char*>(str.c_str());
while (*p_cur != '\0') {
if (sep_idx[(uint8_t)*p_cur] && (!first || !flag)) {
vec.emplace_back(std::string(p_pre, p_cur - p_pre));
p_pre = p_cur + 1;
flag = true;
}
++p_cur;
}
vec.emplace_back(p_pre);
return vec.size();
}
size_t split(const std::string& str, char sep, std::vector<std::string>& vec, bool first)
{
vec.clear();
if (str.empty()) {
return 0;
}
bool flag = false;
char* p_cur = const_cast<char*>(str.c_str());
char* p_pre = const_cast<char*>(str.c_str());
while (*p_cur != '\0') {
if (*p_cur == sep && (!first || !flag)) {
vec.emplace_back(std::string(p_pre, p_cur - p_pre));
p_pre = p_cur + 1;
flag = true;
}
++p_cur;
}
vec.emplace_back(p_pre);
return vec.size();
}
size_t split_by_str(std::string str, const std::string& sep, std::vector<std::string>& vec, bool first)
{
vec.clear();
if (str.empty()) {
return 0;
}
size_t pos = 0;
std::string temp;
while ((pos = str.find(sep)) != std::string::npos) {
temp = str.substr(0, pos);
vec.emplace_back(temp);
str.erase(0, pos + sep.length());
}
vec.emplace_back(str);
return vec.size();
}
void ltrim(std::string& s)
{
s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](unsigned char ch) { return !std::isspace(ch); }));
}
void rtrim(std::string& s)
{
s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) { return !std::isspace(ch); }).base(), s.end());
}
void trim(std::string& s)
{
ltrim(s);
rtrim(s);
}
} // namespace logproxy
} // namespace oceanbase

55
src/common/str.h Normal file
View File

@ -0,0 +1,55 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <string>
#include <vector>
namespace oceanbase {
namespace logproxy {
/**
* copy splitted string to vec
*/
size_t split_all(const std::string& str, const std::string& seps, std::vector<std::string>& vec, bool first = false);
size_t split(const std::string& str, char sep, std::vector<std::string>& vec, bool first = false);
size_t split_by_str(std::string str, const std::string& sep, std::vector<std::string>& vec, bool first = false);
/*
* @param str
* @return null
* @description trim from begin
* @date 2022/9/19 15:36
*/
void ltrim(std::string& s);
/*
* @param str
* @return null
* @description trim from end
* @date 2022/9/19 15:36
*/
void rtrim(std::string& s);
/*
* @param str
* @return null
* @description trim from both begin and end
* @date 2022/9/19 15:36
*/
void trim(std::string& s);
} // namespace logproxy
} // namespace oceanbase

85
src/common/thread.cpp Normal file
View File

@ -0,0 +1,85 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "common/log.h"
#include "common/common.h"
#include "common/thread.h"
namespace oceanbase {
namespace logproxy {
volatile int Thread::_s_tid_idx = 0;
Thread::Thread(const std::string& name) : _name(name)
{
_tid = ::getpid() + OMS_ATOMIC_INC(_s_tid_idx);
}
Thread::~Thread() = default;
void Thread::stop()
{
if (is_run()) {
_run_flag = false;
OMS_DEBUG << "!!! Stop OMS thread: " << _name << "(" << tid() << ")";
}
}
int Thread::join()
{
if (is_run()) {
OMS_DEBUG << "<< Joining thread: " << _name << "(" << tid() << ")";
pthread_join(_thd, nullptr);
OMS_DEBUG << ">> Joined thread: " << _name << "(" << tid() << ")";
}
return _ret;
}
void* Thread::_thd_rotine(void* arg)
{
Thread& thd = *(Thread*)arg;
OMS_DEBUG << "+++ Create thread: " << thd._name << "(" << thd.tid() << ")";
// cast child class poiter to base class pointer
thd.run();
return nullptr;
}
void Thread::start()
{
if (!is_run()) {
set_run(true);
pthread_create(&_thd, nullptr, _thd_rotine, this);
}
}
bool Thread::is_run()
{
return _run_flag;
}
void Thread::set_run(bool run_flag)
{
_run_flag = run_flag;
}
void Thread::detach()
{
pthread_detach(_thd);
}
void Thread::set_ret(int ret)
{
_ret = ret;
}
} // namespace logproxy
} // namespace oceanbase

65
src/common/thread.h Normal file
View File

@ -0,0 +1,65 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <pthread.h>
#include <string>
namespace oceanbase {
namespace logproxy {
class Thread {
public:
explicit Thread(const std::string& name = "");
virtual ~Thread();
virtual void stop();
/**
* @return thread-defined value, default is 0
*/
int join();
void start();
bool is_run();
void set_run(bool run_flag);
void detach();
inline uint32_t tid() const
{
return _tid;
}
protected:
virtual void run() = 0;
void set_ret(int ret);
private:
static void* _thd_rotine(void* arg);
std::string _name;
uint32_t _tid;
pthread_t _thd;
volatile bool _run_flag = false;
int _ret = 0;
static volatile int _s_tid_idx;
};
} // namespace logproxy
} // namespace oceanbase

56
src/common/timer.cpp Normal file
View File

@ -0,0 +1,56 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "timer.h"
namespace oceanbase {
namespace logproxy {
void Timer::sleep(uint64_t interval_us)
{
sleepto(now() + interval_us);
}
void Timer::sleepto(uint64_t nexttime_us)
{
_next_schedule_time_us = nexttime_us;
uint64_t current = now();
while (current < _next_schedule_time_us) {
timeval tv;
timespec timeout;
gettimeofday(&tv, nullptr);
if (tv.tv_usec < 999500) {
timeout.tv_sec = tv.tv_sec;
timeout.tv_nsec = (tv.tv_usec + 500) * 1000;
} else {
timeout.tv_sec = tv.tv_sec + 1;
timeout.tv_nsec = (tv.tv_usec + 500 - 1000000) * 1000;
}
pthread_mutex_lock(&_lock);
pthread_cond_timedwait(&_cond, &_lock, &timeout);
pthread_mutex_unlock(&_lock);
current = now();
}
}
void Timer::interrupt()
{
_next_schedule_time_us = 0;
pthread_cond_signal(&_cond);
}
} // namespace logproxy
} // namespace oceanbase

91
src/common/timer.h Normal file
View File

@ -0,0 +1,91 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <stdint.h>
#include <time.h>
#include <sys/time.h>
#include <pthread.h>
namespace oceanbase {
namespace logproxy {
class Timer {
public:
Timer()
{
reset();
pthread_mutex_init(&_lock, nullptr);
pthread_cond_init(&_cond, nullptr);
}
~Timer()
{
pthread_mutex_destroy(&_lock);
pthread_cond_destroy(&_cond);
}
inline void reset()
{
_start_time_us = now();
}
inline void reset(uint64_t us)
{
_start_time_us = us;
}
static inline uint64_t now()
{
struct timeval tm;
gettimeofday(&tm, nullptr);
return (tm.tv_sec * 1000000 + tm.tv_usec);
}
inline int64_t elapsed() const
{
return now() - _start_time_us;
}
inline int64_t elapsed(uint64_t us) const
{
return us - _start_time_us;
}
inline int64_t stopwatch()
{
reset();
return elapsed();
}
// sleep & interrupt
void sleep(uint64_t interval_us);
void sleepto(uint64_t nexttime_us);
void interrupt();
inline uint64_t start_time() const
{
return _start_time_us;
}
private:
uint64_t _start_time_us = 0;
pthread_mutex_t _lock;
pthread_cond_t _cond;
uint64_t _next_schedule_time_us;
};
} // namespace logproxy
} // namespace oceanbase

17
src/common/version.h Normal file
View File

@ -0,0 +1,17 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#ifndef __OMS_VERSION__
#define __OMS_VERSION__ "2.0.0"
#endif

232
src/communication/channel.h Normal file
View File

@ -0,0 +1,232 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <string>
#include <atomic>
#include <openssl/ssl.h>
#include "event.h"
#include "common/log.h"
#include "common/model.h"
#include "common/config.h"
#include "communication/peer.h"
namespace oceanbase {
namespace logproxy {
class Comm;
class Message;
class Channel {
public:
explicit Channel(const Peer& peer) : _peer(peer)
{
_read_event = static_cast<event*>(malloc(event_get_struct_event_size()));
_read_event->ev_fd = 0;
_write_event = static_cast<event*>(malloc(event_get_struct_event_size()));
_write_event->ev_fd = 0;
}
virtual ~Channel()
{
if (_owned_fd && _peer.fd != 0) {
close(_peer.fd);
OMS_DEBUG << "Closed fd: " << _peer.fd;
}
free(_read_event);
free(_write_event);
}
virtual int after_accept() = 0;
virtual int after_connect() = 0;
/**
* read data from the channel
* @return >0 the bytes has been read
* =0 the connection has been closed or reach the end of file
* <0 error occurs
*/
virtual int read(char* buf, int size) = 0;
/**
* read the specified size of data from the channel
* @return OMS_OK `size` bytes has been read into the buf
* OMS_FAILED some errors occurs
*/
virtual int readn(char* buf, int size) = 0;
/**
* write data to the channel
* @return >0 the bytes has been read
* =0 the connection has been closed
* <0 error occurs
*/
virtual int write(const char* buf, int size) = 0;
/**
* write the specified size of data from the channel
* @return OMS_OK all `size` bytes has been write into the channel
* OMS_FAILED some errors occurs
*/
virtual int writen(const char* buf, int size) = 0;
/**
* Get the last error message.
* It will use the errno internal.
*/
virtual const char* last_error() = 0;
/**
* determin if current channel useable
* if we got nothing from channelFactory through peer-id, we got an DummyChannel which is not ok
*/
virtual bool ok() const
{
return true;
}
inline const Peer& peer() const
{
return _peer;
}
inline void set_communicator(Comm* communicator)
{
_communicator = communicator;
}
inline Comm* communicator()
{
return _communicator;
}
void disable_owned_fd()
{
_owned_fd = false;
}
protected:
friend Comm;
bool _owned_fd = true;
// copy when construct
Peer _peer;
// for Event context
Comm* _communicator = nullptr;
struct event* _read_event = nullptr;
struct event* _write_event = nullptr;
};
class PlainChannel : public Channel {
public:
explicit PlainChannel(const Peer&);
int after_accept() override;
int after_connect() override;
int read(char* buf, int size) override;
int readn(char* buf, int size) override;
int write(const char* buf, int size) override;
int writen(const char* buf, int size) override;
const char* last_error() override;
};
class TlsChannel : public Channel {
public:
explicit TlsChannel(const Peer&);
~TlsChannel() override;
static void close_global();
static int init_global(const Config&);
int init();
int after_accept() override;
int after_connect() override;
int read(char* buf, int size) override;
int readn(char* buf, int size) override;
int write(const char* buf, int size) override;
int writen(const char* buf, int size) override;
const char* last_error() override;
private:
// int verify(int preverify_ok, X509_STORE_CTX* ctx);
static void log_tls_errors();
int handle_error(int ret);
private:
static SSL_CTX* _s_ssl_ctx;
SSL* _ssl = nullptr;
int _last_error = 0;
char _error_string[256];
};
class DummyChannel : public Channel {
OMS_AVOID_COPY(DummyChannel);
public:
DummyChannel() : Channel(Peer(-1))
{}
explicit DummyChannel(const Peer& peer) : Channel(peer)
{}
int after_accept() override
{
return 0;
}
int after_connect() override
{
return 0;
}
int read(char* buf, int size) override
{
return 0;
}
int readn(char* buf, int size) override
{
return 0;
}
int write(const char* buf, int size) override
{
return 0;
}
int writen(const char* buf, int size) override
{
return 0;
}
const char* last_error() override
{
return nullptr;
}
inline bool ok() const override
{
return false;
}
};
} // namespace logproxy
} // namespace oceanbase

127
src/communication/channel_factory.cpp Normal file
View File

@ -0,0 +1,127 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "communication/channel_factory.h"
namespace oceanbase {
namespace logproxy {
static inline Channel* _s_create_plain(const Peer& peer)
{
return new (std::nothrow) PlainChannel(peer);
}
static inline Channel* _s_create_tls(const Peer& peer)
{
return new (std::nothrow) TlsChannel(peer);
}
std::string ChannelFactory::_s_channel_type;
std::function<Channel*(const Peer&)> ChannelFactory::_s_creator;
ChannelFactory::~ChannelFactory()
{
if (_s_channel_type == "tls") {
TlsChannel::close_global();
}
}
int ChannelFactory::init(const Config& config)
{
int ret = OMS_OK;
const std::string& mode = config.communication_mode.val();
if (mode == "server") {
_is_server_mode = true;
} else if (mode == "client") {
_is_server_mode = false;
} else {
OMS_ERROR << "Invalid config of communication_mode: " << mode << ". can be one of server/client";
return OMS_FAILED;
}
OMS_INFO << "ChannelFactory init with " << mode << " mode";
_s_channel_type = config.channel_type.val();
if (_s_channel_type == "plain") {
_s_creator = _s_create_plain;
} else if (_s_channel_type == "tls") {
ret = TlsChannel::init_global(config);
if (ret == OMS_OK) {
_s_creator = _s_create_tls;
}
} else {
OMS_ERROR << "Unsupported channel type: " << _s_channel_type;
ret = OMS_FAILED;
}
return ret;
}
Channel& ChannelFactory::fetch(uint64_t id)
{
auto iter = _channels.find(id);
if (iter == _channels.end()) {
return _dummy;
}
return *(iter->second);
}
Channel& ChannelFactory::add(uint64_t id, const Peer& peer)
{
// const std::lock_guard<std::mutex> lock_guard(_lock);
auto iter = _channels.find(id);
if (iter != _channels.end()) {
OMS_ERROR << "Duplicate channel with id: " << id;
return _dummy;
}
Channel* ch = _s_creator(peer);
if (ch == nullptr) {
OMS_ERROR << "Failed to allocate Channel memory";
return _dummy;
}
_channels.emplace(id, ch);
int ret = _is_server_mode ? ch->after_accept() : ch->after_connect();
if (ret != OMS_OK) {
delete ch;
_channels.erase(id);
return _dummy;
}
return *ch;
}
void ChannelFactory::del(const Channel& channel)
{
auto iter = _channels.find(channel.peer().id());
if (iter != _channels.end()) {
delete iter->second;
_channels.erase(iter);
}
}
void ChannelFactory::clear(int reserved_fd)
{
for (auto& channel : _channels) {
if (reserved_fd != 0 && channel.second != nullptr && channel.second->peer().fd == reserved_fd) {
channel.second->disable_owned_fd();
}
delete channel.second;
}
_channels.clear();
}
} // namespace logproxy
} // namespace oceanbase

61
src/communication/channel_factory.h Normal file
View File

@ -0,0 +1,61 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <mutex>
#include <unordered_map>
#include "communication/channel.h"
namespace oceanbase {
namespace logproxy {
/**
* channel factory owner all channel property,
* all creating, releasing, modifying should invoke
*/
class ChannelFactory {
OMS_SINGLETON(ChannelFactory);
OMS_AVOID_COPY(ChannelFactory);
virtual ~ChannelFactory();
public:
int init(const Config& config);
Channel& fetch(uint64_t id);
Channel& add(uint64_t id, const Peer& peer);
void del(const Channel&);
void clear(int reserved_fd = 0);
inline size_t size()
{
return _channels.size();
}
private:
bool _is_server_mode = true;
std::unordered_map<uint64_t, Channel*> _channels;
static std::string _s_channel_type;
static std::function<Channel*(const Peer&)> _s_creator;
DummyChannel _dummy;
};
} // namespace logproxy
} // namespace oceanbase

424
src/communication/comm.cpp Normal file
View File

@ -0,0 +1,424 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <unistd.h>
#include <assert.h>
#include <arpa/inet.h>
#include <deque>
#include "communication/comm.h"
#include "communication/io.h"
#include "common/counter.h"
namespace oceanbase {
namespace logproxy {
static Config& _s_conf = Config::instance();
MessageDecoder* Comm::_s_decoders[3];
MessageEncoder* Comm::_s_encoders[3];
////////////////////////////////////////////////////////////////////////////////
Comm::Comm()
{
_s_decoders[(int)MessageVersion::V0] = &LegacyDecoder::instance();
_s_decoders[(int)MessageVersion::V1] = &LegacyDecoder::instance();
_s_decoders[(int)MessageVersion::V2] = &ProtobufDecoder::instance();
_s_encoders[(int)MessageVersion::V0] = &LegacyEncoder::instance();
_s_encoders[(int)MessageVersion::V1] = &LegacyEncoder::instance();
_s_encoders[(int)MessageVersion::V2] = &ProtobufEncoder::instance();
// event_enable_debug_logging(EVENT_DBG_ALL);
}
Comm::~Comm()
{
stop();
// FIXME... memory leak, but coredump
// event_base_free(_event_base);
_event_base = nullptr;
}
int Comm::stop(int reserved_fd)
{
close_listen();
_channel_factory.clear(reserved_fd);
if (_event_base == nullptr) {
OMS_INFO << "communication not started";
return OMS_OK;
}
OMS_INFO << "Communicator stopping";
event_base_loopexit(_event_base, nullptr);
OMS_INFO << "Communicator stopped";
return OMS_OK;
}
void Comm::close_listen()
{
if (_listenfd <= 0) {
OMS_WARN << ">>> Communicator disabled listening, fd: " << _listenfd;
close(_listenfd);
_listenfd = 0;
}
}
int Comm::init()
{
if (_event_base != nullptr) {
OMS_WARN << "Communicator has already started";
return OMS_OK;
}
_event_base = event_base_new();
if (_event_base == nullptr) {
OMS_ERROR << "Failed to create event base. system error " << strerror(errno);
return OMS_FAILED;
}
return OMS_OK;
}
int Comm::listen(uint16_t listen_port)
{
if (_listenfd > 0) {
return OMS_OK;
}
_listenfd = oceanbase::logproxy::listen(nullptr, listen_port, false, true);
if (_listenfd <= 0) {
return OMS_FAILED;
}
OMS_INFO << "+++ Listen on port: " << listen_port << ", fd: " << _listenfd;
return OMS_OK;
}
int Comm::start()
{
OMS_INFO << "+++ Communicator about to start";
while (_listenfd > 0) {
int ret = poll();
if (ret != OMS_OK && ret != OMS_AGAIN) {
OMS_ERROR << "Failed to poll communication, ret: " << ret;
break;
}
struct sockaddr_in peer_addr;
socklen_t peer_addr_size = sizeof(struct sockaddr_in);
int connfd = accept(_listenfd, (struct sockaddr*)&peer_addr, &peer_addr_size);
if (connfd > 0) {
_s_evcb_on_accept(connfd, peer_addr);
continue; // may be other connection, dealing directly
}
if (_routine_callback) {
_routine_callback();
}
usleep(_s_conf.accept_interval_us.val());
}
OMS_WARN << "!!! Communicator about to quit";
return OMS_OK;
}
int Comm::poll()
{
int ret = event_base_loop(_event_base, EVLOOP_ONCE | EVLOOP_NONBLOCK);
if (ret == 1) {
return OMS_AGAIN;
}
if (ret != 0) {
OMS_ERROR << "Failed to run Comm, event base dispatch error, ret: " << ret;
return OMS_FAILED;
}
return OMS_OK;
}
void Comm::_s_evcb_on_accept(int fd, const struct sockaddr_in& peer_addr)
{
uint16_t peer_port = ntohs(peer_addr.sin_port);
char peer_host[INET_ADDRSTRLEN];
inet_ntop(PF_INET, &peer_addr.sin_addr, peer_host, sizeof(peer_host));
int ret = set_non_block(fd);
if (ret != OMS_OK) {
OMS_ERROR << "Failed to set non block for fd: " << fd;
close(fd);
return;
}
OMS_DEBUG << "On connect from '" << peer_host << ':' << peer_port << ", fd: " << fd;
Peer peer(peer_addr.sin_addr.s_addr, peer_port, fd);
if (add(peer) != OMS_OK) {
evutil_closesocket(fd);
}
}
int Comm::add(const Peer& peer)
{
uint64_t peer_id = peer.id();
const Channel& exist = _channel_factory.fetch(peer_id);
if (exist.ok()) {
OMS_WARN << "Add channel twice, peer: " << peer.to_string() << ", close last: {}";
// this is unexpected in case of last fd was sent to child process, that current process deleted channel already
// now we try to close last fd first,
// then find child process and stop child process(source reader)
// could be trigger EV_CLOSED after, and release channel there
close(exist.peer().fd);
return OMS_FAILED;
}
Channel& ch = _channel_factory.add(peer_id, peer);
if (!ch.ok()) {
OMS_ERROR << "Failed to create new channel, peer: " << peer.to_string();
return OMS_FAILED;
}
ch.set_communicator(this);
if (_s_conf.verbose.val()) {
OMS_INFO << "Add channel, peer: " << ch.peer().to_string();
}
// we use level trigger as simple, and expect READ first for handshake packet
int ret = event_assign(ch._read_event, _event_base, peer.fd, EV_READ | EV_PERSIST, _s_evcb_on_event, &ch);
if (ret < 0) {
OMS_ERROR << "Failed to do assign event, peer: " << peer.to_string();
_channel_factory.del(ch);
return OMS_FAILED;
}
ret = event_add(ch._read_event, nullptr);
if (ret < 0) {
OMS_ERROR << "Failed to add event, peer: " << peer.to_string();
_channel_factory.del(ch);
return OMS_FAILED;
}
if (_s_conf.verbose.val()) {
OMS_INFO << "Add read channel to Communicator with peer: " << peer.to_string();
}
return OMS_OK;
}
static std::deque<const Message*> _s_msg_queue;
void Comm::_s_evcb_on_event(int fd, short event, void* arg)
{
assert(arg != nullptr);
// Single thread context here
Channel& ch = *(Channel*)arg;
Comm& comm = *ch.communicator();
OMS_DEBUG << "On event fd: " << fd << " got channel, peer: " << ch.peer().to_string();
EventResult err = EventResult::ER_SUCCESS;
if (event & EV_CLOSED) {
OMS_WARN << "On event close, peer: " << ch.peer().to_string();
if (event & EV_READ) {
event_del(ch._read_event);
}
if (event & EV_WRITE) {
event_del(ch._write_event);
}
err = EventResult::ER_CLOSE_CHANNEL;
} else {
if ((event & EV_WRITE)) {
OMS_DEBUG << "On event about to write message: " << ch.peer().to_string();
if (!_s_msg_queue.empty()) {
const Message* msg = _s_msg_queue.front();
_s_msg_queue.pop_front();
if (comm.write_message(ch, *msg) != OMS_OK) {
err = EventResult::ER_CLOSE_CHANNEL;
}
}
event_del(ch._write_event); // one-shot
}
if (event & EV_READ) {
Message* msg = nullptr;
PacketError result = _s_read_message(ch, msg);
if (result == PacketError::SUCCESS) {
if (comm._read_callback) {
err = comm._read_callback(ch.peer(), *msg);
}
} else if (result == PacketError::IGNORE) {
// do nothing
} else {
if (_s_conf.verbose.val()) {
OMS_ERROR << "Failed to handle read message, ret: " << (int)result;
}
err = EventResult::ER_CLOSE_CHANNEL;
}
delete msg;
}
}
if (err == EventResult::ER_CLOSE_CHANNEL) {
if (comm._close_callback) {
comm._close_callback(ch.peer());
}
comm.del(ch);
}
}
/*
*
* =========== Packet Header ============
* [7] magic number
* [2] version
*/
PacketError Comm::_s_read_message(Channel& ch, Message*& msg)
{
uint16_t version;
if (_s_conf.packet_magic.val()) {
char packet_buf[PACKET_MAGIC_SIZE + PACKET_VERSION_SIZE];
if (ch.readn(packet_buf, sizeof(packet_buf)) != OMS_OK) {
OMS_DEBUG << "Failed to read packet magic+version, ch:" << ch.peer().id() << ", error:" << strerror(errno);
return PacketError::NETWORK_ERROR;
}
if (strncmp(packet_buf, PACKET_MAGIC, PACKET_MAGIC_SIZE) != 0) {
OMS_DEBUG << "Invalid packet magic, ignore and close ch:" << ch.peer().id();
return PacketError::PROTOCOL_ERROR;
}
memcpy(&version, packet_buf + PACKET_MAGIC_SIZE, PACKET_VERSION_SIZE);
} else {
if (ch.readn((char*)&version, PACKET_VERSION_SIZE) != OMS_OK) {
OMS_ERROR << "Failed to read packet version, ch:" << ch.peer().id() << ", error:" << strerror(errno);
return PacketError::NETWORK_ERROR;
}
}
version = be_to_cpu(version);
if (!is_version_available(version)) {
OMS_ERROR << "Invalid packet version:" << version << ", ch:" << ch.peer().id();
return PacketError::PROTOCOL_ERROR;
}
auto ret = _s_decoders[version]->decode(ch, (MessageVersion)version, msg);
if (msg != nullptr) {
msg->set_version((MessageVersion)version);
}
return ret;
}
void Comm::del(const Channel& ch)
{
if (ch.ok()) {
OMS_DEBUG << "Try close Channel of peer: " << ch.peer().to_string();
if (ch._read_event != nullptr && ch._read_event->ev_fd != 0) {
event_del(ch._read_event);
}
if (ch._write_event != nullptr && ch._write_event->ev_fd != 0) {
event_del(ch._write_event);
}
_channel_factory.del(ch);
}
}
inline void Comm::del(const Peer& peer)
{
del(_channel_factory.fetch(peer.id()));
}
void Comm::trigger_del(const Peer& peer, const Message& message)
{
send_message(peer, message, true);
// we don't care succ or not, just close
del(peer);
}
int Comm::send_message(const Peer& peer, const Message& msg, bool direct)
{
Channel& ch = _channel_factory.fetch(peer.id());
if (!ch.ok()) {
OMS_ERROR << "Not found channel of peer:" << peer.to_string() << ", just close it";
return OMS_FAILED;
}
if (direct) {
return write_message(ch, msg);
}
int ret = event_assign(ch._write_event, _event_base, peer.fd, EV_WRITE /*| EV_ET*/, _s_evcb_on_event, this);
if (ret < 0) {
OMS_ERROR << "Failed to do event_assign write. fd=" << peer.fd;
return OMS_FAILED;
}
ret = event_add(ch._write_event, nullptr);
if (ret < 0) {
OMS_ERROR << "Failed to do event_add. fd=" << peer.fd;
return OMS_FAILED;
}
return OMS_OK;
}
int Comm::write_message(Channel& ch, const Message& msg)
{
if (_s_conf.verbose_packet.val()) {
OMS_INFO << "About to write mssage: " << msg.debug_string() << ", ch: " << ch.peer().id()
<< ", msg type: " << (int)msg.type();
}
_stage_timer.reset();
size_t raw_len = 0;
MsgBuf buffer;
int ret = _s_encoders[(uint16_t)msg.version()]->encode(msg, buffer, raw_len);
if (ret != OMS_OK) {
OMS_ERROR << "Encoding message failed";
return ret;
}
Counter::instance().count_key(Counter::SENDER_ENCODE_US, _stage_timer.stopwatch());
size_t wsize = 0;
for (const auto& chunk : buffer) {
if (OMS_OK != ch.writen(chunk.buffer(), chunk.size())) {
OMS_ERROR << "Failed to send message through channel:" << ch.peer().id() << ", error:" << ch.last_error();
return OMS_FAILED;
}
wsize += chunk.size();
}
Counter::instance().count_key(Counter::SENDER_SEND_US, _stage_timer.elapsed());
Counter::instance().count_write_io(raw_len);
Counter::instance().count_xwrite_io(wsize);
return OMS_OK;
}
static int debug_events_cb(const struct event_base*, const struct event* ev, void* ctx)
{
OMS_DEBUG << "fd: " << event_get_fd(ev) << ", evflag: " << event_get_events(ev);
return 0;
}
void Comm::debug_events()
{
event_base_foreach_event(_event_base, debug_events_cb, this);
}
} // namespace logproxy
} // namespace oceanbase

119
src/communication/comm.h Normal file
View File

@ -0,0 +1,119 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include <string>
#include <map>
#include <unordered_map>
#include <mutex>
#include <atomic>
#include <functional>
#include "codec/message.h"
#include "communication/channel_factory.h"
#include "codec/encoder.h"
#include "codec/decoder.h"
namespace oceanbase {
namespace logproxy {
enum class EventResult {
ER_SUCCESS = 0,
ER_CLOSE_CHANNEL /// will close the connection
};
class Comm {
public:
Comm();
virtual ~Comm();
int stop(int reserved_fd = 0);
void close_listen();
int init();
int listen(uint16_t listen_port);
int start();
int poll();
int add(const Peer& peer);
void del(const Channel& ch);
void del(const Peer& peer);
void trigger_del(const Peer& peer, const Message& message);
int send_message(const Peer& peer, const Message& msg, bool direct = false);
int write_message(Channel& ch, const Message&);
void debug_events();
inline size_t channel_count()
{
return _channel_factory.size();
}
inline void set_routine_callback(const function<void()>& routine_callback)
{
_routine_callback = routine_callback;
}
inline void set_read_callback(const function<EventResult(const Peer&, const Message&)>& read_callback)
{
_read_callback = read_callback;
}
inline void set_write_callback(const function<EventResult(const Peer&, const Message&)>& write_callback)
{
_write_callback = write_callback;
}
void set_close_callback(const function<void(const Peer&)>& close_callback)
{
_close_callback = close_callback;
}
private:
void _s_evcb_on_accept(int fd, const struct sockaddr_in&);
// void _s_evcb_on_accept_error(struct evconnlistener* listener, void* ctx);
static void _s_evcb_on_event(int fd, short event, void* arg);
static PacketError _s_read_message(Channel& ch, Message*& msg);
private:
Timer _stage_timer;
ChannelFactory& _channel_factory = ChannelFactory::instance();
static MessageDecoder* _s_decoders[3];
static MessageEncoder* _s_encoders[3];
int _listenfd = 0;
// struct evconnlistener* _listener = nullptr;
struct event_base* _event_base = nullptr;
std::function<EventResult(const Peer&, const Message&)> _read_callback;
std::function<EventResult(const Peer&, const Message&)> _write_callback;
std::function<void(const Peer&)> _close_callback;
std::function<void()> _routine_callback;
};
} // namespace logproxy
} // namespace oceanbase

158
src/communication/http.cpp Normal file
View File

@ -0,0 +1,158 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <sys/queue.h>
#include "event2/event.h"
#include "event2/buffer.h"
#include "event2/dns.h"
#include "event2/keyvalq_struct.h"
#include "common/common.h"
#include "common/log.h"
#include "common/guard.hpp"
#include "communication/http.h"
namespace oceanbase {
namespace logproxy {
struct HttpContext {
struct event_base* base;
// error of libevent
evhttp_request_error error;
HttpResponse* response = nullptr;
};
static void http_request_done(struct evhttp_request*, void* arg)
{
event_base_loopbreak(((HttpContext*)arg)->base);
}
static int http_header_cb(struct evhttp_request* response, void* arg)
{
HttpContext* context = (HttpContext*)arg;
context->response->code = evhttp_request_get_response_code(response);
context->response->message = evhttp_request_get_response_code_line(response);
struct evkeyval* header;
struct evkeyvalq* headers = evhttp_request_get_input_headers(response);
TAILQ_FOREACH(header, headers, next)
{
context->response->headers.emplace(header->key, header->value);
}
return 0;
}
static void http_payload_cb(struct evhttp_request* reponse, void* arg)
{
HttpContext* context = (HttpContext*)arg;
int n = 0;
char buf[8192];
struct evbuffer* evbuf = evhttp_request_get_input_buffer(reponse);
while ((n = evbuffer_remove(evbuf, buf, sizeof(buf))) > 0) {
context->response->payload.append(buf, n);
}
}
static void http_request_error_cb(enum evhttp_request_error error, void* arg)
{
OMS_ERROR << "HTTP request error:" << error;
HttpContext* context = (HttpContext*)arg;
context->error = error;
event_base_loopexit(context->base, nullptr);
}
void http_conn_close_cb(struct evhttp_connection*, void* arg)
{
event_base_loopexit(((HttpContext*)arg)->base, nullptr);
}
int HttpClient::get(const std::string& url, HttpResponse& response)
{
evhttp_uri* uri = evhttp_uri_parse(url.c_str());
if (uri == nullptr) {
OMS_ERROR << "Failed to http GET, failed to parse url:" << url.c_str();
return OMS_FAILED;
}
FreeGuard<evhttp_uri*> uri_fg(uri, evhttp_uri_free);
const char* host = evhttp_uri_get_host(uri);
if (host == nullptr) {
OMS_ERROR << "Failed to http GET, failed to parse host";
return OMS_FAILED;
}
int port = evhttp_uri_get_port(uri);
if (port == -1) {
port = 80;
}
const char* path = evhttp_uri_get_path(uri);
std::string request_url = (path == nullptr || strlen(path) == 0) ? "/" : path;
const char* query = evhttp_uri_get_query(uri);
if (query != nullptr && strlen(query) > 0) {
request_url.append("?").append(query);
}
struct event_base* base = event_base_new();
FreeGuard<event_base*> base_fg(base, event_base_free);
if (base == nullptr) {
OMS_ERROR << "Failed to http GET, failed to event_base_new";
return OMS_FAILED;
}
struct evdns_base* dnsbase = evdns_base_new(base, 1);
if (dnsbase == nullptr) {
OMS_WARN << "Failed to invoke evdns_base_new, just skip";
}
FreeGuard<evdns_base*> dnsbase_fg(dnsbase, [](struct evdns_base* ptr) { evdns_base_free(ptr, 0); });
HttpContext context;
context.base = base;
context.response = &response;
struct evhttp_connection* conn = evhttp_connection_base_new(base, dnsbase, host, port);
FreeGuard<evhttp_connection*> conn_fg(conn, evhttp_connection_free);
if (conn == nullptr) {
OMS_ERROR << "Failed to http GET, failed to evhttp_connection_base";
return OMS_FAILED;
}
evhttp_connection_set_timeout(conn, 600);
evhttp_connection_set_closecb(conn, http_conn_close_cb, &context);
struct evhttp_request* req = evhttp_request_new(http_request_done, &context);
if (req == nullptr) {
OMS_ERROR << "Failed to http GET, failed to evhttp_request";
return OMS_FAILED;
}
evhttp_request_set_header_cb(req, http_header_cb);
evhttp_request_set_chunked_cb(req, http_payload_cb);
evhttp_request_set_error_cb(req, http_request_error_cb);
evhttp_add_header(evhttp_request_get_output_headers(req), "Host", host);
OMS_DEBUG << "url:" << url << " host:" << host << " port:" << port << " path:" << path
<< " request_url:" << request_url;
// The connection gets ownership of the request
int ret = evhttp_make_request(conn, req, EVHTTP_REQ_GET, request_url.c_str());
if (ret == -1) {
OMS_ERROR << "Failed to http GET of make request";
return OMS_FAILED;
}
// start invoke http
event_base_dispatch(base);
return OMS_OK;
}
} // namespace logproxy
} // namespace oceanbase

33
src/communication/http.h Normal file
View File

@ -0,0 +1,33 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include "event2/http.h"
namespace oceanbase {
namespace logproxy {
struct HttpResponse {
int code;
std::string message;
std::map<std::string, std::string> headers;
std::string payload;
};
class HttpClient {
public:
static int get(const std::string& url, HttpResponse& response);
};
} // namespace logproxy
} // namespace oceanbase

255
src/communication/io.cpp Normal file
View File

@ -0,0 +1,255 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <sys/socket.h>
#include <unistd.h>
#include <errno.h>
#include <fcntl.h>
#include <string.h>
#include <netdb.h>
#include <poll.h>
#include "communication/io.h"
#include "common/log.h"
#include "common/common.h"
namespace oceanbase {
namespace logproxy {
int writen(int fd, const void* buf, int size)
{
const char* tmp = (const char*)buf;
while (size > 0) {
// OMS_INFO << "about to write, fd: " << fd << ", size: " << size;
const ssize_t ret = ::write(fd, tmp, size);
// OMS_INFO << "done to write, fd:" << fd << ", size: " << size << ", ret:" << ret << ", errno:" << errno;
if (ret >= 0) {
tmp += ret;
size -= ret;
continue;
}
const int err = errno;
if (EAGAIN != err && EINTR != err) {
return OMS_FAILED;
}
}
return OMS_OK;
}
int readn(int fd, void* buf, int size)
{
char* tmp = (char*)buf;
while (size > 0) {
// OMS_INFO << "about to read, fd: " << fd << ", size: " << size;
const ssize_t ret = ::read(fd, tmp, size);
// OMS_INFO << "done to read, fd:" << fd << ", size: " << size << ", ret:" << ret << ", errno:" << errno;
if (ret > 0) {
tmp += ret;
size -= ret;
continue;
}
if (0 == ret) {
return OMS_FAILED; // end of file
}
const int err = errno;
if (EAGAIN != err && EINTR != err) {
return OMS_FAILED;
}
}
return OMS_OK;
}
int connect(const char* host, int port, bool block_mode, int timeout, int& sockfd)
{
if (nullptr == host || 0 == host[0] || port < 0 || port >= 65536) {
OMS_ERROR << "Invalid host or port";
return OMS_FAILED;
}
struct hostent* hostent = gethostbyname(host);
if (hostent == nullptr) {
OMS_ERROR << "Failed to get host by name(" << host << "). error=" << strerror(errno);
return OMS_FAILED;
}
struct sockaddr_in serv_addr;
memset(&serv_addr, 0, sizeof(serv_addr));
serv_addr.sin_family = PF_INET;
serv_addr.sin_port = htons(port);
serv_addr.sin_addr = *((struct in_addr*)hostent->h_addr);
const int sock = socket(PF_INET, SOCK_STREAM, 0);
if (sock == -1) {
OMS_ERROR << "Failed to create socket. error=" << strerror(errno);
return OMS_FAILED;
}
int ret = OMS_OK;
if (!block_mode) {
ret = set_non_block(sock);
if (ret != 0) {
close(sock);
return OMS_FAILED;
}
}
ret = connect(sock, (struct sockaddr*)&serv_addr, sizeof(serv_addr));
if (ret != 0) {
if (!block_mode && (errno == EINPROGRESS || errno == EAGAIN)) {
struct pollfd pollfd;
pollfd.fd = sock;
pollfd.events = POLLERR | POLLHUP | POLLOUT;
pollfd.revents = 0;
ret = poll(&pollfd, 1, timeout);
if (ret == 1 && (pollfd.revents & POLLOUT)) {
// connect success
OMS_DEBUG << "Connect to server success after poll. host=" << host << ",port=" << port;
} else {
OMS_ERROR << "Failed to connect to server. host=" << host << ",port=" << port
<< ". timeout:" << (bool)(ret == 0) << ", error=" << strerror(errno);
close(sock);
return OMS_CONNECT_FAILED;
}
} else {
OMS_ERROR << "Failed to connect to server. host=" << host << ",port=" << port << ". error=" << strerror(errno);
close(sock);
return OMS_CONNECT_FAILED;
}
}
sockfd = sock;
return OMS_OK;
}
int listen(const char* host, int port, bool block_mode, bool reuse_address)
{
if (port < 0 || port > 65536) {
OMS_ERROR << "Invalid listen port: " << port;
return OMS_FAILED;
}
struct in_addr bind_addr;
bind_addr.s_addr = INADDR_ANY;
if (host != nullptr) {
struct hostent* hostent = gethostbyname(host);
if (nullptr == hostent) {
OMS_ERROR << "gethostbyname return failed. address=" << host << ", error=" << strerror(errno);
return OMS_FAILED;
}
bind_addr = *(struct in_addr*)hostent->h_addr;
} else {
host = "(not-set)"; // print log
}
struct sockaddr_in sockaddr;
memset(&sockaddr, 0, sizeof(sockaddr));
sockaddr.sin_family = PF_INET;
sockaddr.sin_port = htons(port);
sockaddr.sin_addr = bind_addr;
int sock = socket(PF_INET, SOCK_STREAM, 0);
if (sock < 0) {
OMS_ERROR << "Failed to create socket. error=" << strerror(errno);
return OMS_FAILED;
}
int ret = OMS_OK;
if (reuse_address) {
ret = set_reuse_addr(sock);
if (ret != OMS_OK) {
::close(sock);
return ret;
}
}
ret = bind(sock, (struct sockaddr*)&sockaddr, sizeof(sockaddr));
if (ret != OMS_OK) {
OMS_ERROR << "Failed to bind address '" << host << ":" << port << "', error: " << strerror(errno);
::close(sock);
return OMS_FAILED;
}
ret = ::listen(sock, 10);
if (ret != OMS_OK) {
OMS_ERROR << "Failed to listen on '" << host << ':' << port << "', error: " << strerror(errno);
::close(sock);
return OMS_FAILED;
}
if (!block_mode) {
ret = set_non_block(sock);
if (ret != OMS_OK) {
OMS_ERROR << "Failed to set listen socket non-block mode, error: " << strerror(errno);
::close(sock);
return OMS_FAILED;
}
}
ret = set_close_on_exec(sock);
if (ret != OMS_OK) {
OMS_ERROR << "Failed to set listen socket close on exec mode, error: " << strerror(errno);
::close(sock);
return OMS_FAILED;
}
return sock;
}
int set_reuse_addr(int sock)
{
int opt = 1;
int ret = setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt));
if (ret != 0) {
OMS_ERROR << "Failed to set socket in 'REUSE_ADDR' mode. error=" << strerror(errno);
return OMS_FAILED;
}
return OMS_OK;
}
int set_non_block(int fd)
{
int flags = fcntl(fd, F_GETFL);
if (flags == -1) {
OMS_WARN << "Failed to get flags of fd(" << fd << "). error=" << strerror(errno);
return OMS_FAILED;
}
flags = fcntl(fd, F_SETFL, flags | O_NONBLOCK);
if (flags == -1) {
OMS_WARN << "Failed to set non-block flags of fd(" << fd << "). error=" << strerror(errno);
return OMS_FAILED;
}
return OMS_OK;
}
int set_close_on_exec(int fd)
{
int flags = fcntl(fd, F_GETFL);
if (flags == -1) {
OMS_WARN << "Failed to get flags of fd(" << fd << "). error=" << strerror(errno);
return OMS_FAILED;
}
flags = fcntl(fd, F_SETFL, flags | O_CLOEXEC);
if (flags == -1) {
OMS_WARN << "Failed to set close on exec flags of fd(" << fd << "). error=" << strerror(errno);
return OMS_FAILED;
}
return OMS_OK;
}
} // namespace logproxy
} // namespace oceanbase

49
src/communication/io.h Normal file
View File

@ -0,0 +1,49 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
namespace oceanbase {
namespace logproxy {
int writen(int fd, const void* buf, int size);
int readn(int fd, void* buf, int size);
/**
* create a socket and connect to remote server
* @param host the hostname of remote server
* @param port the port of server
* @param block_mode the blocking mode of socket
* @param timeout The timeout of connect. It will worked only when `block_mode=false`
* -1 means infinite, 0 means no wait, others means the time to wait, in millisecond
* @param sockfd[out] The socket file descriptor if the connection is OK.
* @return OMS_OK if connect success
*/
int connect(const char* host, int port, bool block_mode, int timeout, int& sockfd);
/**
* create a socket, bind the address and listen on it
* @param host The hostname to bind. If null, then bind ANY_ADDRESS
* @param port The port to bind
* @param block_mode the blocking mode of socket
* @param reuse_address Whether to set option SO_REUSEADDR
* @return sockfd If success, return the socket descriptor
*/
int listen(const char* host, int port, bool block_mode, bool reuse_address);
int set_reuse_addr(int fd);
int set_non_block(int fd);
int set_close_on_exec(int fd);
} // namespace logproxy
} // namespace oceanbase

0
src/communication/listener.cpp Normal file
View File

0
src/communication/listener.h Normal file
View File

95
src/communication/peer.h Normal file
View File

@ -0,0 +1,95 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include "common/timer.h"
namespace oceanbase {
namespace logproxy {
struct Peer {
public:
int fd = 0;
public:
explicit Peer(in_addr_t a = 0, in_port_t p = 0, int fd = 0) : fd(fd)
{
// HIGH
// ^
// | 32bit: peer ip
// | 16bit: peer port
// | 16bit: fd & UINT16_MAX
// LOW
_id = (((uint64_t)a) << 32);
_id |= (uint32_t(p) << 16);
_id |= (fd & UINT16_MAX);
_addr = a;
_port = p;
}
Peer(const Peer& rhs) = default;
Peer(Peer&& rhs) noexcept = default;
Peer& operator=(const Peer& rhs) = default;
Peer& operator=(Peer&& rhs) = default;
inline uint64_t id() const
{
return _id;
}
inline bool operator==(const Peer& other) const
{
return this->_id == other._id;
}
inline bool operator!=(const Peer& other) const
{
return this->_id != other._id;
}
inline bool operator<(const Peer& rhs) const
{
return _id < rhs._id;
}
inline std::size_t operator()(const Peer& p) const
{
return p._id;
}
inline friend LogStream& operator<<(LogStream& ss, const Peer& o)
{
ss << "fd:" << o.fd;
return ss;
}
std::string to_string() const
{
LogStream ls{0, "", 0, nullptr};
ls << "id:" << _id << ", fd:" << fd << ", addr:" << _addr << ", port:" << _port;
return ls.str();
}
private:
uint64_t _id;
in_addr_t _addr;
in_port_t _port;
};
} // namespace logproxy
} // namespace oceanbase

58
src/communication/plain_channel.cpp Normal file
View File

@ -0,0 +1,58 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "communication/io.h"
#include "communication/channel.h"
#include "communication/comm.h"
namespace oceanbase {
namespace logproxy {
PlainChannel::PlainChannel(const Peer& peer) : Channel(peer)
{}
int PlainChannel::after_accept()
{
return OMS_OK;
}
int PlainChannel::after_connect()
{
return OMS_OK;
}
int PlainChannel::read(char* buf, int size)
{
return ::read(_peer.fd, buf, size);
}
int PlainChannel::readn(char* buf, int size)
{
return ::oceanbase::logproxy::readn(_peer.fd, buf, size);
}
int PlainChannel::write(const char* buf, int size)
{
return ::write(_peer.fd, buf, size);
}
int PlainChannel::writen(const char* buf, int size)
{
return ::oceanbase::logproxy::writen(_peer.fd, buf, size);
}
const char* PlainChannel::last_error()
{
return strerror(errno);
}
} // namespace logproxy
} // namespace oceanbase

335
src/communication/tls_channel.cpp Normal file
View File

@ -0,0 +1,335 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <assert.h>
#include <openssl/rsa.h>
#include <openssl/crypto.h>
#include <openssl/err.h>
#include "communication/channel.h"
namespace oceanbase {
namespace logproxy {
SSL_CTX* TlsChannel::_s_ssl_ctx = nullptr;
TlsChannel::TlsChannel(const Peer& peer) : Channel(peer)
{
init();
}
TlsChannel::~TlsChannel()
{
if (_ssl != nullptr) {
SSL_free(_ssl);
_ssl = nullptr;
}
}
/**
* The SSL will call this routine while verify a peer connection.
* You can print some logs about the context
* @param preverify_ok 1 for ok and 0 verify failed
* @param ctx verify context
* @return return 1 for verify ok and 0 verify failed
*/
static int verify_callback(int preverify_ok, X509_STORE_CTX* ctx)
{
X509* cert = X509_STORE_CTX_get_current_cert(ctx);
if (nullptr == cert) {
return preverify_ok;
}
int error = X509_STORE_CTX_get_error(ctx);
SSL* ssl = (SSL*)X509_STORE_CTX_get_ex_data(ctx, SSL_get_ex_data_X509_STORE_CTX_idx());
if (nullptr == ssl) {
return preverify_ok;
}
if (1 != preverify_ok) {
char buf[256];
X509_NAME_oneline(X509_get_subject_name(cert), buf, sizeof(buf));
OMS_ERROR << "verify failed. error " << error << ":" << X509_verify_cert_error_string(error)
<< ", subject name:" << buf;
if (error == X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT) {
X509_NAME_oneline(X509_get_issuer_name(cert), buf, sizeof(buf));
OMS_ERROR << "issuer:" << buf;
}
}
return preverify_ok;
/**
TlsChannel* channel = static_cast<TlsChannel*>(SSL_get_ex_data(ssl));
if (nullptr == channel) {
return preverify_ok;
}
return channel->verify(preverify_ok, ctx);
*/
}
void TlsChannel::close_global()
{
if (_s_ssl_ctx != nullptr) {
SSL_CTX_free(_s_ssl_ctx);
_s_ssl_ctx = nullptr;
}
}
int TlsChannel::init_global(const Config& config)
{
const char* ca_cert_file = config.tls_ca_cert_file.val().c_str();
const char* cert_file = config.tls_cert_file.val().c_str();
const char* key_file = config.tls_key_file.val().c_str();
bool verify_peer = config.tls_verify_peer.val();
SSL_load_error_strings();
OpenSSL_add_ssl_algorithms();
// const SSL_METHOD * ssl_method = TLS_server_method(); // openssl version 1.1.1 supported
const SSL_METHOD* ssl_method = SSLv23_method(); // TLSv1.3 not supptorted
SSL_CTX* ssl_ctx = SSL_CTX_new(ssl_method);
if (ssl_ctx == nullptr) {
OMS_ERROR << "Failed to create SSL_CTX";
return OMS_FAILED;
}
int mode = verify_peer ? SSL_VERIFY_PEER : SSL_VERIFY_NONE;
SSL_CTX_set_verify(ssl_ctx, mode, verify_callback);
int ret = SSL_CTX_load_verify_locations(ssl_ctx, ca_cert_file, nullptr);
if (ret != 1) {
OMS_ERROR << "SSL_CTX_load_verify_locations failed. ca cert file: '" << ca_cert_file << '\'';
log_tls_errors();
SSL_CTX_free(ssl_ctx);
return OMS_FAILED;
}
ret = SSL_CTX_use_certificate_file(ssl_ctx, cert_file, SSL_FILETYPE_PEM);
if (ret != 1) {
OMS_ERROR << "SSL_CTX_use_certificate_file failed. cert file: '" << cert_file << '\'';
log_tls_errors();
SSL_CTX_free(ssl_ctx);
return OMS_FAILED;
}
ret = SSL_CTX_use_PrivateKey_file(ssl_ctx, key_file, SSL_FILETYPE_PEM);
if (ret != 1) {
OMS_ERROR << "SSL_CTX_use_PrivateKey_file failed. key file: '" << key_file << '\'';
log_tls_errors();
SSL_CTX_free(ssl_ctx);
return OMS_FAILED;
}
ret = SSL_CTX_check_private_key(ssl_ctx);
if (ret != 1) {
OMS_ERROR << "SSL_CTX_check_private_key failed. ca cert file: '" << ca_cert_file << "', cert file: '" << cert_file
<< "', key file: '" << key_file << '\'';
log_tls_errors();
SSL_CTX_free(ssl_ctx);
return OMS_FAILED;
}
_s_ssl_ctx = ssl_ctx;
return OMS_OK;
}
int TlsChannel::init()
{
assert(_s_ssl_ctx != nullptr);
SSL* ssl = SSL_new(_s_ssl_ctx);
if (ssl == nullptr) {
OMS_ERROR << "Failed to create ssl";
return OMS_FAILED;
}
int ret = SSL_set_fd(ssl, _peer.fd);
if (ret != 1) {
log_tls_errors();
SSL_free(ssl);
return OMS_FAILED;
}
_ssl = ssl;
return OMS_OK;
}
void TlsChannel::log_tls_errors()
{
unsigned long error = ERR_get_error();
char buf[256];
while (error != 0) {
ERR_error_string_n(error, buf, sizeof(buf));
OMS_ERROR << std::string(buf);
error = ERR_get_error();
}
}
int TlsChannel::after_accept()
{
// refer https://www.openssl.org/docs/manmaster/man3/SSL_accept.html
// if fd is nonblocking, SSL_accept may return a negative number
while (true) {
const int ret = SSL_accept(_ssl);
if (ret < 0) {
const int error = SSL_get_error(_ssl, ret);
if (error == SSL_ERROR_WANT_READ || error == SSL_ERROR_WANT_WRITE || error == SSL_ERROR_WANT_ACCEPT) {
usleep(1000 * 10); // TODO instead with poll
continue;
} else {
OMS_ERROR << "Failed to accept new connection. error=" << error;
log_tls_errors();
return OMS_FAILED;
}
} else if (ret == 0) {
OMS_ERROR << "Failed to accept new connection. error is 0";
log_tls_errors();
return OMS_FAILED;
} else if (ret == 1) {
OMS_DEBUG << "accept new connection success";
return OMS_OK;
}
}
return OMS_FAILED;
}
int TlsChannel::after_connect() // TODO simplify with after_accept
{
// refer https://www.openssl.org/docs/manmaster/man3/SSL_connect.html
// if fd is nonblocking, SSL_accept may return a negative number
while (true) {
const int ret = SSL_connect(_ssl);
if (ret < 0) {
const int error = SSL_get_error(_ssl, ret);
if (error == SSL_ERROR_WANT_READ || error == SSL_ERROR_WANT_WRITE || error == SSL_ERROR_WANT_ACCEPT) {
usleep(1000 * 10); // TODO instead with poll
continue;
} else {
OMS_ERROR << "Failed to connect. error=" << error;
log_tls_errors();
return OMS_FAILED;
}
} else if (ret == 0) {
OMS_ERROR << "Failed to connect. error is 0";
log_tls_errors();
return OMS_FAILED;
} else if (ret == 1) {
OMS_DEBUG << "connect to server success";
return OMS_OK;
}
}
return OMS_FAILED;
}
int TlsChannel::read(char* buf, int size)
{
ERR_clear_error();
int ret = SSL_read(_ssl, buf, size);
if (ret > 0) {
return ret;
}
return handle_error(ret);
}
int TlsChannel::readn(char* buf, int size)
{
ERR_clear_error();
_last_error = 0;
while (size > 0) {
int ret = SSL_read(_ssl, buf, size);
if (ret > 0) {
size -= ret;
buf += ret;
continue;
}
_last_error = SSL_get_error(_ssl, ret);
if (_last_error != SSL_ERROR_WANT_READ) {
return OMS_FAILED;
}
}
return OMS_OK;
}
int TlsChannel::write(const char* buf, int size)
{
ERR_clear_error();
int ret = SSL_write(_ssl, buf, size);
if (ret > 0) {
return ret;
}
return handle_error(ret);
}
int TlsChannel::writen(const char* buf, int size)
{
ERR_clear_error();
_last_error = 0;
while (size > 0) {
int ret = SSL_write(_ssl, buf, size);
if (ret > 0) {
size -= ret;
buf += ret;
continue;
}
_last_error = SSL_get_error(_ssl, ret);
if (_last_error != SSL_ERROR_WANT_WRITE) {
return OMS_FAILED;
}
}
return OMS_OK;
}
int TlsChannel::handle_error(int ret)
{
const int error = SSL_get_error(_ssl, ret);
_last_error = error;
if (error == SSL_ERROR_ZERO_RETURN) {
return 0;
}
return -1;
}
const char* TlsChannel::last_error()
{
switch (_last_error) {
case SSL_ERROR_NONE:
return "None";
case SSL_ERROR_ZERO_RETURN:
return "The TLS/SSL connection has been closed";
case SSL_ERROR_WANT_READ:
case SSL_ERROR_WANT_WRITE:
return "The operation did not complete";
case SSL_ERROR_WANT_CONNECT:
case SSL_ERROR_WANT_ACCEPT:
return "The operation did not complete";
case SSL_ERROR_WANT_X509_LOOKUP:
return "The TLS/SSL I/O function should be called again later";
case SSL_ERROR_SYSCALL: {
snprintf(_error_string, sizeof(_error_string), "System error:%s", strerror(errno));
return _error_string;
}
case SSL_ERROR_SSL: {
unsigned long error = ERR_get_error();
ERR_error_string_n(error, _error_string, sizeof(_error_string));
return _error_string;
}
default: {
snprintf(_error_string, sizeof(_error_string), "Unknown error:%d", _last_error);
return _error_string;
}
}
}
} // namespace logproxy
} // namespace oceanbase

194
src/demo/client_demo.cpp Normal file
View File

@ -0,0 +1,194 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase Migration Service LogProxy is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "common/log.h"
#include "common/config.h"
#include "common/option.h"
#include "codec/encoder.h"
#include "communication/io.h"
#include "communication/comm.h"
#include "obaccess/oblog_config.h"
#include "obaccess/mysql_protocol.h"
using namespace oceanbase::logproxy;
void debug_record(const RecordDataMessage& record)
{
OMS_INFO << "fetch a record from liboblog: ";
// << "record_type: " << record.recordType()
// << ", timestamp: " << record->getTimestamp()
// << ", checkpoint: " << record->getFileNameOffset()
// << ", dbname: " << record->dbname()
// << ", tbname: " << record->tbname();
}
EventResult on_msg(const Peer& peer, const Message& message)
{
switch (message.type()) {
case MessageType::HANDSHAKE_RESPONSE_CLIENT:
OMS_INFO << "Connect LogProxy Succ, response: " << ((ClientHandshakeResponseMessage&)message).to_string();
break;
case MessageType::DATA_CLIENT: {
const RecordDataMessage& record = (const RecordDataMessage&)message;
OMS_INFO << "data packet, compress: " << (int)record.compress_type << ", count: " << record.count();
debug_record(record);
break;
}
case MessageType::STATUS:
break;
case MessageType::ERROR_RESPONSE:
OMS_ERROR << "Error occured: " << ((ErrorMessage&)message).to_string();
return EventResult::ER_CLOSE_CHANNEL;
default:
OMS_WARN << "Unknown Event income: " << (int)message.type();
break;
}
return EventResult::ER_SUCCESS;
}
EventResult on_err(const Peer& peer, PacketError err)
{
OMS_ERROR << "Error occured peer: " << peer.to_string() << ", err: " << (int)err;
return EventResult::ER_CLOSE_CHANNEL;
}
int run(const std::string& host, uint16_t port, const std::string& client_id, const std::string& config)
{
Comm comm;
int ret = comm.init();
if (ret != OMS_OK) {
OMS_ERROR << "Failed to init Communicator";
return ret;
}
int sockfd = 0;
ret = connect(host.c_str(), port, true, 0, sockfd);
if (ret != OMS_OK || sockfd <= 0) {
OMS_ERROR << "Failed to connect " << host << ":" << port;
return -1;
}
set_non_block(sockfd);
OMS_INFO << "Connected to " << host << ":" << port << " with sockfd: " << sockfd;
struct sockaddr_in peer_addr;
socklen_t len;
ret = getpeername(sockfd, (struct sockaddr*)&peer_addr, &len);
if (ret == 0 && peer_addr.sin_addr.s_addr != 0) {
Peer p(peer_addr.sin_addr.s_addr, ntohs(peer_addr.sin_port), sockfd);
OMS_INFO << "fetched peer: " << p.to_string();
} else {
OMS_WARN << "Failed to fetch peer info of fd:" << sockfd << ", errno:" << errno << ", error:" << strerror(errno);
}
Peer peer(peer_addr.sin_addr.s_addr, htons(peer_addr.sin_port), sockfd);
// ret = comm.add(peer);
// if (ret != OMS_OK) {
// OMS_ERROR << "Failed to add channel with sockfd: " << sockfd << ", ret: " << ret;
// return -1;
// }
//
ClientHandshakeRequestMessage handshake((int)LogType::OCEANBASE,
host.c_str(),
client_id.empty() ? "DemoClient" : client_id.c_str(),
"1.0.0",
false,
config.c_str());
comm.listen(port + 1);
ret = comm.start();
ret = comm.send_message(peer, handshake);
if (ret != OMS_OK) {
OMS_ERROR << "Failed to send handshake: " << handshake.to_string();
return -1;
}
comm.set_read_callback(on_msg);
return ret;
}
int main(int argc, char** argv)
{
std::string host = "127.0.0.1";
uint16_t port = 2983;
std::string config;
std::string client_id;
std::string channel_type = "plain";
std::string tls_ca_cert_file;
std::string tls_key_file;
std::string tls_cert_file;
OmsOptions options("LogProxy Client Demo");
options.add(OmsOption('h', "host", true, "Host", [&](const std::string& optarg) { host = optarg; }));
options.add(OmsOption(
'P', "port", true, "Port", [&](const std::string& optarg) { port = strtol(optarg.c_str(), nullptr, 10); }));
options.add(OmsOption('c', "config", true, "Configuration", [&](const std::string& optarg) { config = optarg; }));
options.add(OmsOption('i', "id", true, "Client Id", [&](const std::string& optarg) { client_id = optarg; }));
options.add(OmsOption(
'T', "channel-type", true, "Channel-Type(plain/tls)", [&](const std::string& optarg) { channel_type = optarg; }));
options.add(
OmsOption('C', "tls-ca-file", true, "ca file", [&](const std::string& optarg) { tls_ca_cert_file = optarg; }));
options.add(
OmsOption('E', "tls-cert-file", true, "cert file", [&](const std::string& optarg) { tls_cert_file = optarg; }));
options.add(
OmsOption('K', "tls-key-file", true, "key file", [&](const std::string& optarg) { tls_key_file = optarg; }));
struct option* long_options = options.long_pattern();
std::string&& pattern = options.pattern();
for (int opt = 0; (opt = getopt_long(argc, argv, pattern.c_str(), long_options, nullptr)) != -1;) {
OmsOption* option = options.get(opt);
if (option == nullptr) {
options.usage();
return 0;
} else {
option->func(option->is_arg ? std::string(optarg) : "");
}
}
if (config.empty()) {
printf("No configuration, please use -c\n");
return -1;
}
// init_log(argv[0]);
OblogConfig oblog_config(config);
if (!oblog_config.password.val().empty()) {
std::string password_sha1;
MysqlProtocol::do_sha_password(oblog_config.password.val(), password_sha1);
oblog_config.password.set(dumphex(password_sha1));
}
if (!oblog_config.sys_password.val().empty()) {
// std::string sys_password_sha1;
// MysqlProtocol::do_sha_password(oblog_config.sys_password.val(), sys_password_sha1);
oblog_config.sys_password.set(oblog_config.sys_password.val());
}
Config::instance().verbose.set(true);
Config::instance().verbose_packet.set(true);
Config::instance().communication_mode.set("client");
Config::instance().channel_type.set(channel_type);
Config::instance().tls_key_file.set(tls_key_file);
Config::instance().tls_cert_file.set(tls_cert_file);
Config::instance().tls_ca_cert_file.set(tls_ca_cert_file);
Config::instance().packet_magic.set(false);
int ret = ChannelFactory::instance().init(Config::instance());
if (ret != OMS_OK) {
OMS_ERROR << "Failed to init channel factory";
return OMS_FAILED;
}
return run(host, port, client_id, oblog_config.generate_config_str());
}

Some files were not shown because too many files have changed in this diff Show More