Mysql元数据导入Apache Atlas
薛钧
通过Rest API实现Mysql元数据写入Apache Atlas
请求方式：POST
请求路径：http://hostname:21000/api/atlas/v2/entity
在atlas中，有多种实体类型定义，也可以自定义实体类型，本示例展示的是通过rdbms类型创建mysql元数据。数据库实例使用rdbms_instance类型、数据库使用rdbms_db类型、表使用rdbms_table类型、列使用rdbms_column类型。/v2/entity接口每次只创建一个实体，批量创建可以使用/v2/entity/bulk接口。atlas中通过json文件对实体类型进行定义，通过查看源码addons/models/2000-RDBMS/2010-rdbms_model.json可看到rdbms类型的定义。
创建数据源实例
rdbms_instance类型继承DataSet类型，attributeDefs中定义了该类型的属性信息，isOptional代表该属性是否是选填项，在创建实体时一定要填写必填项，否则在创建实体时会提示缺失mandatory attribute value missing in type，同时要注意父类型中的必填项。isUnique代表该属性是否唯一。
{
"name": "rdbms_instance",
"description": "Instance that the rdbms server is running on",
"superTypes": ["DataSet"],
"serviceType": "rdbms",
"typeVersion": "1.1",
"attributeDefs": [
{
"name": "rdbms_type",
"typeName": "string",
"isOptional": false,
"cardinality": "SINGLE",
"isUnique": false,
"isIndexable": true
},
{
"name": "platform",
"typeName": "string",
"isOptional": true,
"cardinality": "SINGLE",
"isUnique": false,
"isIndexable": true
},
{
"name": "cloudOrOnPrem",
"typeName": "string",
"isOptional": true,
"cardinality": "SINGLE",
"isUnique": false,
"isIndexable": false
},
{
"name": "hostname",
"typeName": "string",
"isOptional": true,
"cardinality": "SINGLE",
"isUnique": false,
"isIndexable": false
},
{
"name": "port",
"typeName": "int",
"isOptional": true,
"cardinality": "SINGLE",
"isUnique": false,
"isIndexable": false
},
{
"name": "protocol",
"typeName": "string",
"isOptional": true,
"cardinality": "SINGLE",
"isUnique": false,
"isIndexable": false
},
{
"name": "contact_info",
"typeName": "string",
"isOptional": true,
"cardinality": "SINGLE",
"isUnique": false,
"isIndexable": false
},
{
"name": "comment",
"typeName": "string",
"isOptional": true,
"cardinality": "SINGLE",
"isUnique": false,
"isIndexable": false
}
]
}
example
{
"entity":{
"typeName":"rdbms_instance",
"attributes":{
"qualifiedName":"192.168.2.11@mysql",
"name":"192.168.2.11@mysql",
"rdbms_type":"MYSQL",
"platform":"CentOS 7.5",
"cloudOrOnPrem":"cloud",
"hostname":"192.168.2.11",
"port":"3306",
"protocol":"http",
"contact_info":"jdbc",
"comment":"rdbms_instance API insert test",
"description":"rdbms_instance test",
"owner":"root"
}
}
}
[root@atlas bin]# curl -v -u admin:admin -X POST -H "Content-Type:application/json" -d '{"entity":{"typeName":"rdbms_instance","attributes":{"qualifiedName":"192.168.2.11@mysql","name":"192.168.2.11@mysql","rdbms_type":"MYSQL","platform":"CentOS 7.5","cloudOrOnPrem":"cloud","hostname":"192.168.2.11","port":"3306","protocol":"http","contact_info":"jdbc","comment":"rdbms_instance API insert test","description":"rdbms_instance test","owner":"root"}}}' "http://atlas :21000/api/atlas/v2/entity"
* About to connect() to atlas port 21000 (#0)
* Trying 192.168.2.11...
* Connected to atlas (192.168.2.11) port 21000 (#0)
* Server auth using Basic with user 'admin'
> POST /api/atlas/v2/entity HTTP/1.1
> Authorization: Basic YWRtaW46YWRtaW4=
> User-Agent: curl/7.29.0
> Host: atlas:21000
> Accept: */*
> Content-Type:application/json
> Content-Length: 363
>
* upload completely sent off: 363 out of 363 bytes
< HTTP/1.1 200 OK
< Date: Sun, 28 Jun 2020 16:22:16 GMT
< Set-Cookie: ATLASSESSIONID=kt4u9zdyhjrb1aa8ierpllgpu;Path=/;HttpOnly
< Expires: Thu, 01 Jan 1970 00:00:00 GMT
< X-Frame-Options: DENY
< X-Content-Type-Options: nosniff
< X-XSS-Protection: 1; mode=block
< Strict-Transport-Security: max-age=31536000; includeSubDomains
< Content-Type: application/json; charset=UTF-8
< X-Content-Type-Options: nosniff
< Content-Security-Policy: default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' blob: data:; connect-src 'self'; img-src 'self' blob: data:; style-src 'self' 'unsafe-inline';font-src 'self' data:
< Server: Apache Atlas
< Transfer-Encoding: chunked
<
* Connection #0 to host atlas left intact
{"mutatedEntities":{"CREATE":[{"typeName":"rdbms_instance","attributes":{"qualifiedName":"192.168.2.11@mysql"},"guid":"b68383ce-e9f9-4ac8-9731-b70622eb355b"}]},"guidAssignments":{"-420268170623097":"b68383ce-e9f9-4ac8-9731-b70622eb355b"}}
返回消息中有创建实体的guid信息，我们刚刚创建的mysql数据库实例的guid是b68383ce-e9f9-4ac8-9731-b70622eb355b，guid是全局唯一的标识，在创建数据库时填写数据库实例的guid可以对两者进行关联。还可以通过guid对实体进行查询。
创建数据库实例，并关联数据源实例
rdbms_db类型同样继承自DataSet。
{
"name": "rdbms_db",
"description": "a database (schema) in an rdbms",
"superTypes": ["DataSet"],
"serviceType": "rdbms",
"typeVersion": "1.1",
"attributeDefs": [
{
"name": "prodOrOther",
"typeName": "string",
"isOptional": true,
"cardinality": "SINGLE",
"isUnique": false,
"isIndexable": true
},
{
"name": "contact_info",
"typeName": "string",
"isOptional": true,
"cardinality": "SINGLE",
"isUnique": false,
"isIndexable": false
}
]
}
{
"entity":{
"typeName":"rdbms_db",
"attributes":{
"qualifiedName":"fyb@192.168.2.11@mysql",
"owner":"root",
"ownerType":"USER",
"name":"fyb",
"emailAddress":"root",
"createdBy":"root",
"createTime":"2020-06-28 10:18:46.555Z",
"updatedBy":"root",
"updateTime":"2020-06-28 10:18:46.666Z",
"description":"rdbms_db fyb API input",
"prodOrOther":"asd",
"version":0,
"contact_info":"qwe",
"instance":{
"guid":"b68383ce-e9f9-4ac8-9731-b70622eb355b",
"typeName":"rdbms_instance"
}
}
}
}
[root@atlas bin]# curl -v -u admin:admin -X POST -H "Content-Type:application/json" -d '{"entity":{"typeName":"rdbms_db","attributes":{"qualifiedName":"fyb@192.168.2.11@mysql","owner":"root","ownerType":"USER","name":"fyb","emailAddress":"root","createdBy":"root","createTime":"2020-06-28 10:18:46.555Z","updatedBy":"root","updateTime":"2020-06-28 10:18:46.666Z","description":"rdbms_db fyb API input","prodOrOther":"asd","version":0,"contact_info":"qwe","instance":{"guid":"b68383ce-e9f9-4ac8-9731-b70622eb355b","typeName":"rdbms_instance"}}}}' "http://atlas:21000/api/atlas/v2/entity"
* About to connect() to atlas port 21000 (#0)
* Trying 192.168.2.11...
* Connected to atlas (192.168.2.11) port 21000 (#0)
* Server auth using Basic with user 'admin'
> POST /api/atlas/v2/entity HTTP/1.1
> Authorization: Basic YWRtaW46YWRtaW4=
> User-Agent: curl/7.29.0
> Host: atlas:21000
> Accept: */*
> Content-Type:application/json
> Content-Length: 456
>
* upload completely sent off: 456 out of 456 bytes
< HTTP/1.1 200 OK
< Date: Sun, 28 Jun 2020 16:29:28 GMT
< Set-Cookie: ATLASSESSIONID=1suf3omtstenfk81tc1g98jjp;Path=/;HttpOnly
< Expires: Thu, 01 Jan 1970 00:00:00 GMT
< X-Frame-Options: DENY
< X-Content-Type-Options: nosniff
< X-XSS-Protection: 1; mode=block
< Strict-Transport-Security: max-age=31536000; includeSubDomains
< Content-Type: application/json; charset=UTF-8
< X-Content-Type-Options: nosniff
< Content-Security-Policy: default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' blob: data:; connect-src 'self'; img-src 'self' blob: data:; style-src 'self' 'unsafe-inline';font-src 'self' data:
< Server: Apache Atlas
< Transfer-Encoding: chunked
<
* Connection #0 to host atlas left intact
{"mutatedEntities":{"UPDATE":[{"typeName":"rdbms_instance","attributes":{"owner":"root","qualifiedName":"192.168.2.11@mysql","name":"192.168.2.11@mysql","description":"rdbms_instance test"},"guid":"b68383ce-e9f9-4ac8-9731-b70622eb355b","status":"ACTIVE","displayText":"192.168.2.11@mysql","classificationNames":[],"meaningNames":[],"meanings":[]}],"CREATE":[{"typeName":"rdbms_db","attributes":{"qualifiedName":"fyb@192.168.2.11@mysql"},"guid":"5972becf-c042-45a8-9156-1f32115f9fe6"}]},"guidAssignments":{"-420268170623100":"5972becf-c042-45a8-9156-1f32115f9fe6"}}
创建表实例
{
"name": "rdbms_table",
"description": "a table in an rdbms database (schema)",
"superTypes": ["DataSet"],
"serviceType": "rdbms",
"typeVersion": "1.2",
"options": {
"schemaElementsAttribute": "columns"
},
"attributeDefs": [
{
"name": "name_path",
"typeName": "string",
"isOptional": true,
"cardinality": "SINGLE",
"isUnique": false,
"isIndexable": false
},
{
"name": "createTime",
"typeName": "date",
"isOptional": true,
"cardinality": "SINGLE",
"isUnique": false,
"isIndexable": false
},
{
"name": "comment",
"typeName": "string",
"isOptional": true,
"cardinality": "SINGLE",
"isUnique": false,
"isIndexable": false
},
{
"name": "type",
"typeName": "string",
"isOptional": true,
"cardinality": "SINGLE",
"isUnique": false,
"isIndexable": false
},
{
"name": "contact_info",
"typeName": "string",
"isOptional": true,
"cardinality": "SINGLE",
"valuesMinCount": 1,
"valuesMaxCount": 1,
"isUnique": false,
"isIndexable": false
}
]
}
{
"entity":{
"typeName":"rdbms_table",
"attributes":{
"qualifiedName":"fyb.haha@192.168.2.11@mysql",
"createdBy":"root",
"name":"haha",
"comment":"rdbms_table API insert test",
"description":"rdbms_table 手动输入",
"owner":"root",
"type":"table",
"contact_info":"fyb_info",
"db":{
"guid":"5972becf-c042-45a8-9156-1f32115f9fe6",
"typeName":"rdbms_db"
}
}
}
}
[root@atlas bin]# curl -v -u admin:admin -X POST -H "Content-Type:application/json" -d '{"entity":{"typeName":"rdbms_table","attributes":{"qualifiedName":"fyb.haha@192.168.2.11@mysql","createdBy":"root","name":"haha","comment":"rdbms_table API insert test","description":"rdbms_table 手动输入","owner":"root","type":"table","contact_info":"table_01_info","db":{"guid":"5972becf-c042-45a8-9156-1f32115f9fe6","typeName":"rdbms_db"}}}}' "http://atlas:21000/api/atlas/v2/entity"
* About to connect() to atlas port 21000 (#0)
* Trying 192.168.2.11...
* Connected to atlas (192.168.2.11) port 21000 (#0)
* Server auth using Basic with user 'admin'
> POST /api/atlas/v2/entity HTTP/1.1
> Authorization: Basic YWRtaW46YWRtaW4=
> User-Agent: curl/7.29.0
> Host: atlas:21000
> Accept: */*
> Content-Type:application/json
> Content-Length: 349
>
* upload completely sent off: 349 out of 349 bytes
< HTTP/1.1 200 OK
< Date: Mon, 29 Jun 2020 01:51:42 GMT
< Set-Cookie: ATLASSESSIONID=6rhcaqu554wx14ugizdodwu6b;Path=/;HttpOnly
< Expires: Thu, 01 Jan 1970 00:00:00 GMT
< X-Frame-Options: DENY
< X-Content-Type-Options: nosniff
< X-XSS-Protection: 1; mode=block
< Strict-Transport-Security: max-age=31536000; includeSubDomains
< Content-Type: application/json; charset=UTF-8
< X-Content-Type-Options: nosniff
< Content-Security-Policy: default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' blob: data:; connect-src 'self'; img-src 'self' blob: data:; style-src 'self' 'unsafe-inline';font-src 'self' data:
< Server: Apache Atlas
< Transfer-Encoding: chunked
<
* Connection #0 to host atlas left intact
{"mutatedEntities":{"UPDATE":[{"typeName":"rdbms_db","attributes":{"owner":"root","qualifiedName":"fyb@192.168.2.11@mysql","name":"fyb","description":"rdbms_db fyb API input"},"guid":"5972becf-c042-45a8-9156-1f32115f9fe6","status":"ACTIVE","displayText":"fyb","classificationNames":[],"meaningNames":[],"meanings":[]}],"CREATE":[{"typeName":"rdbms_table","attributes":{"qualifiedName":"fyb.haha@192.168.2.11@mysql"},"guid":"06cccc33-8c39-4889-ae63-feefe0d8249c"}]},"guidAssignments":{"-422563721869641":"06cccc33-8c39-4889-ae63-feefe0d8249c"}}
创建字段实例
{
"name": "rdbms_column",
"description": "a column in an rdbms table",
"superTypes": ["DataSet"],
"serviceType": "rdbms",
"typeVersion": "1.2",
"options": {
"schemaAttributes": "[\"name\", \"description\", \"owner\", \"data_type\", \"comment\", \" isPrimaryKey\", \" isNullable\"]"
},
"attributeDefs": [
{
"name": "data_type",
"typeName": "string",
"isOptional": true,
"cardinality": "SINGLE",
"isUnique": false,
"isIndexable": true
},
{
"name": "length",
"typeName": "int",
"isOptional": true,
"cardinality": "SINGLE",
"isUnique": false,
"isIndexable": false
},
{
"name": "default_value",
"typeName": "string",
"isOptional": true,
"cardinality": "SINGLE",
"isUnique": false,
"isIndexable": false
},
{
"name": "comment",
"typeName": "string",
"isOptional": true,
"cardinality": "SINGLE",
"isUnique": false,
"isIndexable": false
},
{
"name": "isNullable",
"typeName": "boolean",
"isOptional": true,
"cardinality": "SINGLE",
"isUnique": false,
"isIndexable": false
},
{
"name": "isPrimaryKey",
"typeName": "boolean",
"isOptional": true,
"cardinality": "SINGLE",
"isUnique": false,
"isIndexable": false
}
]
}
{
"entity":{
"typeName":"rdbms_column",
"attributes":{
"qualifiedName":"fyb.haha.id@192.168.2.11@mysql",
"createdBy":"root",
"name":"id",
"comment":"rdbms_column API insert test",
"description":"rdbms_column API insert test",
"owner":"root",
"data_type":"string",
"length":20,
"default_value":0,
"isNullable":true,
"isPrimaryKey":false,
"contact_info":"id_info",
"table":{
"guid":"06cccc33-8c39-4889-ae63-feefe0d8249c",
"typeName":"rdbms_table"
}
}
}
}
[root@atlas bin]# curl -v -u admin:admin -X POST -H "Content-Type:application/json" -d '{"entity":{"typeName":"rdbms_column","attributes":{"qualifiedName":"fyb.haha.id@192.168.2.11@mysql","createdBy":"root","name":"id","comment":"rdbms_column API insert test","description":"rdbms_column API insert test","owner":"root","data_type":"string","length":20,"default_value":0,"isNullable":true,"isPrimaryKey":false,"contact_info":"id_info","table":{"guid":"06cccc33-8c39-4889-ae63-feefe0d8249c","typeName":"rdbms_table"}}}}' "http://atlas:21000/api/atlas/v2/entity"
* About to connect() to atlas port 21000 (#0)
* Trying 192.168.2.11...
* Connected to atlas (192.168.2.11) port 21000 (#0)
* Server auth using Basic with user 'admin'
> POST /api/atlas/v2/entity HTTP/1.1
> Authorization: Basic YWRtaW46YWRtaW4=
> User-Agent: curl/7.29.0
> Host: atlas:21000
> Accept: */*
> Content-Type:application/json
> Content-Length: 430
>
* upload completely sent off: 430 out of 430 bytes
< HTTP/1.1 200 OK
< Date: Mon, 29 Jun 2020 01:59:54 GMT
< Set-Cookie: ATLASSESSIONID=ins06hsestd4qyscoc8f0y9r;Path=/;HttpOnly
< Expires: Thu, 01 Jan 1970 00:00:00 GMT
< X-Frame-Options: DENY
< X-Content-Type-Options: nosniff
< X-XSS-Protection: 1; mode=block
< Strict-Transport-Security: max-age=31536000; includeSubDomains
< Content-Type: application/json; charset=UTF-8
< X-Content-Type-Options: nosniff
< Content-Security-Policy: default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' blob: data:; connect-src 'self'; img-src 'self' blob: data:; style-src 'self' 'unsafe-inline';font-src 'self' data:
< Server: Apache Atlas
< Transfer-Encoding: chunked
<
* Connection #0 to host atlas left intact
{"mutatedEntities":{"UPDATE":[{"typeName":"rdbms_table","attributes":{"owner":"root","createTime":0,"qualifiedName":"fyb.haha@192.168.2.11@mysql","name":"haha","description":"rdbms_table 手动输入"},"guid":"06cccc33-8c39-4889-ae63-feefe0d8249c","status":"ACTIVE","displayText":"haha","classificationNames":[],"meaningNames":[],"meanings":[]}],"CREATE":[{"typeName":"rdbms_column","attributes":{"qualifiedName":"fyb.haha.id@192.168.2.11@mysql"},"guid":"b4d37e2f-6c13-4533-a05c-09c53bdd6452"}]},"guidAssignments":{"-422563721869653":"b4d37e2f-6c13-4533-a05c-09c53bdd6452"}}
该示例只是演示下如何创建实体，以及创建实体时需要注意的一些事项，rdbms类型详细定义可以参考源码。
类似资料：