Ambari数据库分析-配置相关

最新推荐文章于 2023-04-26 11:14:16 发布

worgent

最新推荐文章于 2023-04-26 11:14:16 发布

阅读量2.1k

点赞数 1

分类专栏： ambari 文章标签：数据库 ambari 配置

本文链接：https://blog.csdn.net/tianbaochao/article/details/72677607

版权

ambari 专栏收录该内容

22 篇文章 1 订阅

订阅专栏

重点介绍配置相关存储。
主要是服务对应的配置问题;主机对应的配置问题;用户修改的配置问题.

clusterconfig

集群配置表，包含各种服务组件的相关配置

41  version1    1   core-site   2   2   {"fs.defaultFS":"hdfs://hadoop181:8020","ipc.server.tcpnodelay":"true","mapreduce.jobtracker.webinterface.trusted":"false","hadoop.security.auth_to_local":"DEFAULT","ipc.client.idlethreshold":"8000","hadoop.proxyuser.hdfs.groups":"*","fs.trash.interval":"360","hadoop.security.authorization":"false","hadoop.http.authentication.simple.anonymous.allowed":"true","ipc.client.connection.maxidletime":"30000","hadoop.proxyuser.hcat.groups":"*","hadoop.proxyuser.hive.hosts":"hadoop182","ha.failover-controller.active-standby-elector.zk.op.retries":"120","hadoop.proxyuser.ambari.hosts":"hadoop181","hadoop.security.authentication":"simple","hadoop.proxyuser.hdfs.hosts":"*","hadoop.proxyuser.ambari.groups":"*","ipc.client.connect.max.retries":"50","io.file.buffer.size":"131072","hadoop.proxyuser.hcat.hosts":"hadoop182","net.topology.script.file.name":"/etc/hadoop/conf/topology_script.py","io.compression.codecs":"org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec","hadoop.proxyuser.hive.groups":"*","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization"}  {"final":{"fs.defaultFS":"true"}}   1489658974532
65  version1    1   yarn-site   2   2   {"yarn.resourcemanager.zk-address":"hadoop183:2181,hadoop182:2181,hadoop181:2181","yarn.timeline-service.entity-group-fs-store.active-dir":"/ats/active/","yarn.nodemanager.container-monitor.interval-ms":"3000","yarn.nodemanager.linux-container-executor.group":"hadoop","yarn.application.classpath":"$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*","yarn.admin.acl":"activity_analyzer,yarn","yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds":"3600","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.address":"0.0.0.0:45454","yarn.resourcemanager.webapp.address":"hadoop182:8088","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","yarn.scheduler.maximum-allocation-vcores":"3","yarn.timeline-service.address":"hadoop182:10200","yarn.timeline-service.enabled":"true","yarn.nodemanager.aux-services":"mapreduce_shuffle,spark_shuffle,spark2_shuffle","yarn.nodemanager.resource.cpu-vcores":"3","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"false","yarn.nodemanager.aux-services.spark_shuffle.class":"org.apache.spark.network.yarn.YarnShuffleService","yarn.nodemanager.log-aggregation.debug-enabled":"false","yarn.resourcemanager.zk-acl":"world:anyone:rwcda","yarn.http.policy":"HTTP_ONLY","yarn.client.nodemanager-connect.max-wait-ms":"60000","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","yarn.resourcemanager.admin.address":"hadoop182:8141","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","yarn.resourcemanager.resource-tracker.address":"hadoop182:8025","yarn.resourcemanager.fs.state-store.retry-policy-spec":"2000, 500","yarn.nodemanager.delete.debug-delay-sec":"0","yarn.timeline-service.http-authentication.type":"simple","yarn.timeline-service.ttl-enable":"true","yarn.timeline-service.entity-group-fs-store.retain-seconds":"604800","yarn.log-aggregation.retain-seconds":"2592000","yarn.nodemanager.resource.memory-mb":"4096","yarn.timeline-service.http-authentication.proxyuser.ambari.groups":"*","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","yarn.node-labels.enabled":"false","yarn.nodemanager.log.retain-second":"604800","yarn.resourcemanager.connect.max-wait.ms":"900000","yarn.scheduler.minimum-allocation-vcores":"1","yarn.resourcemanager.zk-num-retries":"1000","yarn.nodemanager.aux-services.spark2_shuffle.classpath":"{{stack_root}}/${hdp.version}/spark2/aux/*","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.resourcemanager.system-metrics-publisher.enabled":"true","yarn.timeline-service.client.max-retries":"30","yarn.timeline-service.client.retry-interval-ms":"1000","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.EntityGroupFSTimelineStore","yarn.timeline-service.bind-host":"0.0.0.0","yarn.nodemanager.container-metrics.unregister-delay-ms":"60000","yarn.timeline-service.generic-application-history.store-class":"org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore","yarn.resourcemanager.fs.state-store.uri":" ","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"hadoop-yarn","yarn.resourcemanager.recovery.enabled":"true","yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath":"","yarn.resourcemanager.scheduler.address":"hadoop182:8030","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","yarn.timeline-service.ttl-ms":"2678400000","yarn.nodemanager.health-checker.script.timeout-ms":"60000","yarn.nodemanager.aux-services.spark2_shuffle.class":"org.apache.spark.network.yarn.YarnShuffleService","yarn.log.server.url":"http://hadoop182:19888/jobhistory/logs","yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes":"org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl","yarn.resourcemanager.address":"hadoop182:8050","yarn.timeline-service.recovery.enabled":"true","yarn.resourcemanager.scheduler.monitor.enable":"false","yarn.nodemanager.recovery.dir":"{{yarn_log_dir_prefix}}/nodemanager/recovery-state","hadoop.registry.rm.enabled":"true","yarn.resourcemanager.bind-host":"0.0.0.0","yarn.nodemanager.log-aggregation.num-log-files-per-app":"30","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX\u003d$MALLOC_ARENA_MAX","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","yarn.acl.enable":"false","yarn.resourcemanager.ha.enabled":"false","yarn.client.nodemanager-connect.retry-interval-ms":"10000","yarn.timeline-service.version":"1.5","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","yarn.nodemanager.local-dirs":"/hadoop/yarn/local","yarn.timeline-service.leveldb-timeline-store.path":"/hadoop/yarn/timeline","yarn.nodemanager.recovery.enabled":"true","yarn.resourcemanager.zk-timeout-ms":"10000","yarn.resourcemanager.am.max-attempts":"2","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.node-labels.fs-store.root-dir":"/system/yarn/node-labels","yarn.node-labels.fs-store.retry-policy-spec":"2000, 500","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.resourcemanager.work-preserving-recovery.enabled":"true","yarn.log-aggregation-enable":"true","yarn.nodemanager.aux-services.spark_shuffle.classpath":"{{stack_root}}/${hdp.version}/spark/aux/*","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore","yarn.timeline-service.leveldb-state-store.path":"/hadoop/yarn/timeline","yarn.timeline-service.entity-group-fs-store.done-dir":"/ats/done/","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90","hadoop.registry.zk.quorum":"hadoop183:2181,hadoop182:2181,hadoop181:2181","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"3600","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","yarn.resourcemanager.nodes.exclude-path":"/etc/hadoop/conf/yarn.exclude","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.timeline-service.http-authentication.proxyuser.ambari.hosts":"hadoop181","yarn.nodemanager.bind-host":"0.0.0.0","yarn.resourcemanager.hostname":"hadoop182","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.timeline-service.webapp.address":"hadoop182:8188","yarn.scheduler.minimum-allocation-mb":"512","yarn.timeline-service.entity-group-fs-store.summary-store":"org.apache.hadoop.yarn.server.timeline.RollingLevelDBTimelineStore","yarn.nodemanager.health-checker.interval-ms":"135000","yarn.resourcemanager.zk-retry-interval-ms":"1000","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"1000","yarn.nodemanager.remote-app-log-dir":"/app-logs","yarn.scheduler.maximum-allocation-mb":"4096","yarn.nodemanager.vmem-check-enabled":"false","yarn.timeline-service.entity-group-fs-store.scan-interval-seconds":"60","yarn.resourcemanager.webapp.https.address":"hadoop182:8090","yarn.timeline-service.webapp.https.address":"hadoop182:8190","yarn.nodemanager.resource.percentage-physical-cpu-limit":"80","yarn.nodemanager.vmem-pmem-ratio":"2.1","yarn.nodemanager.log-aggregation.compression-type":"gz","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","yarn.nodemanager.log-dirs":"/hadoop/yarn/log"} {}  1489658977060

script

CREATE TABLE clusterconfig (
  config_id BIGINT NOT NULL,
  version_tag VARCHAR(100) NOT NULL,
  version BIGINT NOT NULL,
  type_name VARCHAR(100) NOT NULL,
  cluster_id BIGINT NOT NULL,
  stack_id BIGINT NOT NULL,
  selected SMALLINT NOT NULL DEFAULT 0,
  config_data LONGTEXT NOT NULL,
  config_attributes LONGTEXT,
  create_timestamp BIGINT NOT NULL,
  selected_timestamp BIGINT NOT NULL DEFAULT 0,
  CONSTRAINT PK_clusterconfig PRIMARY KEY (config_id),
  CONSTRAINT FK_clusterconfig_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id),
  CONSTRAINT FK_clusterconfig_stack_id FOREIGN KEY (stack_id) REFERENCES stack(stack_id),
  CONSTRAINT UQ_config_type_tag UNIQUE (cluster_id, type_name, version_tag),
  CONSTRAINT UQ_config_type_version UNIQUE (cluster_id, type_name, version));

重点说明

/**
- The most recent time that this configuration was marked as
- {@link #selected}. This is useful when configruations are being reverted
- since a reversion does not create a new instance. Another configuration may
- technically be newer via its creation date ({@link #timestamp}), however
- that does not indicate it was the most recently enabled configuration.
  */

serviceconfig

服务配置，通过serviceconfigmapping关联表与clusterconfig相关联。

CREATE TABLE serviceconfig (
  service_config_id BIGINT NOT NULL,
  cluster_id BIGINT NOT NULL,
  service_name VARCHAR(255) NOT NULL,
  version BIGINT NOT NULL,
  create_timestamp BIGINT NOT NULL,
  stack_id BIGINT NOT NULL,
  user_name VARCHAR(255) NOT NULL DEFAULT '_db',
  group_id BIGINT,
  note LONGTEXT,
  CONSTRAINT PK_serviceconfig PRIMARY KEY (service_config_id),
  CONSTRAINT FK_serviceconfig_stack_id FOREIGN KEY (stack_id) REFERENCES stack(stack_id),
  CONSTRAINT UQ_scv_service_version UNIQUE (cluster_id, service_name, version));

　serviceconfigmapping

关联service配置表与cluster配置表

serviceconfighosts

关联servic配置表与主机表

configgroup

配置组,用于保存用户自定义配置.

2   2   tianbaochao YARN    New configuration group created on Fri Mar 17 2017  1489716220882   YARN

CREATE TABLE configgroup (
  group_id BIGINT,
  cluster_id BIGINT NOT NULL,
  group_name VARCHAR(255) NOT NULL,
  tag VARCHAR(1024) NOT NULL,
  description VARCHAR(1024),
  create_timestamp BIGINT NOT NULL,
  service_name VARCHAR(255),
  CONSTRAINT PK_configgroup PRIMARY KEY (group_id),
  CONSTRAINT FK_configgroup_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id));

configgrouphostmapping

配置组和主机映射关系

CREATE TABLE configgrouphostmapping (
  config_group_id BIGINT NOT NULL,
  host_id BIGINT NOT NULL,
  CONSTRAINT PK_configgrouphostmapping PRIMARY KEY (config_group_id, host_id),
  CONSTRAINT FK_cghm_cgid FOREIGN KEY (config_group_id) REFERENCES configgroup (group_id),
  CONSTRAINT FK_cghm_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id));

confgroupclusterconfigmapping

配置组与集群配置映射关系

数据

2   2   yarn-site   version1493012679312    _db 1493012680084

script

CREATE TABLE confgroupclusterconfigmapping (
  config_group_id BIGINT NOT NULL,
  cluster_id BIGINT NOT NULL,
  config_type VARCHAR(100) NOT NULL,
  version_tag VARCHAR(100) NOT NULL,
  user_name VARCHAR(100) DEFAULT '_db',
  create_timestamp BIGINT NOT NULL,
  CONSTRAINT PK_confgroupclustercfgmapping PRIMARY KEY (config_group_id, cluster_id, config_type),
  CONSTRAINT FK_cgccm_gid FOREIGN KEY (config_group_id) REFERENCES configgroup (group_id),
  CONSTRAINT FK_confg FOREIGN KEY (cluster_id, config_type, version_tag) REFERENCES clusterconfig (cluster_id, type_name, version_tag));

hostconfigmapping

/**
* Entity that represents a host config mapping and override.
*/
主机

CREATE TABLE hostconfigmapping (
  create_timestamp BIGINT NOT NULL,
  host_id BIGINT NOT NULL,
  cluster_id BIGINT NOT NULL,
  type_name VARCHAR(255) NOT NULL,
  selected INTEGER NOT NULL DEFAULT 0,
  service_name VARCHAR(255),
  version_tag VARCHAR(255) NOT NULL,
  user_name VARCHAR(255) NOT NULL DEFAULT '_db',
  CONSTRAINT PK_hostconfigmapping PRIMARY KEY (create_timestamp, host_id, cluster_id, type_name),
  CONSTRAINT FK_hostconfmapping_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id),
  CONSTRAINT FK_hostconfmapping_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id));