Ranger Hive plugin 源代码分析

最新推荐文章于 2023-10-09 10:01:24 发布
最新推荐文章于 2023-10-09 10:01:24 发布
阅读量1.9k 收藏 2
点赞数 1
分类专栏： hive 文章标签： hive
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/houzhizhen/article/details/120435985
版权

hiveserver2-site.xml

<property>
  <name>hive.security.authorization.enabled</name>
  <value>true</value>
</property>

<property>
  <name>hive.security.authorization.manager</name>
  <value>org.apache.ranger.authorization.hive.authorizer.RangerHiveAuthorizerFactory</value>
</property>

RangerHiveAuthorizerFactory 实现了 HiveAuthorizerFactory。

public class RangerHiveAuthorizerFactory implements HiveAuthorizerFactory {
	@Override
	public HiveAuthorizer createHiveAuthorizer(HiveMetastoreClientFactory metastoreClientFactory,
											   HiveConf                   conf,
											   HiveAuthenticationProvider hiveAuthenticator,
											   HiveAuthzSessionContext    sessionContext)
													   throws HiveAuthzPluginException {
		return new RangerHiveAuthorizer(metastoreClientFactory, conf, hiveAuthenticator, sessionContext);
	}
}

RangerHiveAuthorizerFactory 的 createHiveAuthorizer 方法返回 HiveAuthorizer，里面定义了Hive 权限的 plugin实现必须实现的方法

public interface HiveAuthorizer {
    HiveAuthorizer.VERSION getVersion();

    void grantPrivileges(List<HivePrincipal> var1, List<HivePrivilege> var2, HivePrivilegeObject var3, HivePrincipal var4, boolean var5) throws HiveAuthzPluginException, HiveAccessControlException;

    void revokePrivileges(List<HivePrincipal> var1, List<HivePrivilege> var2, HivePrivilegeObject var3, HivePrincipal var4, boolean var5) throws HiveAuthzPluginException, HiveAccessControlException;

    void createRole(String var1, HivePrincipal var2) throws HiveAuthzPluginException, HiveAccessControlException;

    void dropRole(String var1) throws HiveAuthzPluginException, HiveAccessControlException;

    List<HiveRoleGrant> getPrincipalGrantInfoForRole(String var1) throws HiveAuthzPluginException, HiveAccessControlException;

    List<HiveRoleGrant> getRoleGrantInfoForPrincipal(HivePrincipal var1) throws HiveAuthzPluginException, HiveAccessControlException;

    void grantRole(List<HivePrincipal> var1, List<String> var2, boolean var3, HivePrincipal var4) throws HiveAuthzPluginException, HiveAccessControlException;

    void revokeRole(List<HivePrincipal> var1, List<String> var2, boolean var3, HivePrincipal var4) throws HiveAuthzPluginException, HiveAccessControlException;

    void checkPrivileges(HiveOperationType var1, List<HivePrivilegeObject> var2, List<HivePrivilegeObject> var3, HiveAuthzContext var4) throws HiveAuthzPluginException, HiveAccessControlException;

    List<HivePrivilegeObject> filterListCmdObjects(List<HivePrivilegeObject> var1, HiveAuthzContext var2) throws HiveAuthzPluginException, HiveAccessControlException;

    List<String> getAllRoles() throws HiveAuthzPluginException, HiveAccessControlException;

    List<HivePrivilegeInfo> showPrivileges(HivePrincipal var1, HivePrivilegeObject var2) throws HiveAuthzPluginException, HiveAccessControlException;

    void setCurrentRole(String var1) throws HiveAccessControlException, HiveAuthzPluginException;

    List<String> getCurrentRoleNames() throws HiveAuthzPluginException;

    void applyAuthorizationConfigPolicy(HiveConf var1) throws HiveAuthzPluginException;

    Object getHiveAuthorizationTranslator() throws HiveAuthzPluginException;

    List<HivePrivilegeObject> applyRowFilterAndColumnMasking(HiveAuthzContext var1, List<HivePrivilegeObject> var2) throws SemanticException;

    boolean needTransform();

    HivePolicyProvider getHivePolicyProvider() throws HiveAuthzPluginException;

    public static enum VERSION {
        V1;

        private VERSION() {
        }
    }
}

RangerHiveAuthorizer 的集成关系

RangerHiveAuthorizer 不是直接实现 HiveAuthorizer。RangerHiveAuthorizer 继承RangerHiveAuthorizerBase。RangerHiveAuthorizerBase 继承 AbstractHiveAuthorizer。AbstractHiveAuthorizer 实现 HiveAuthorizer。

AbstractHiveAuthorizer

public abstract class AbstractHiveAuthorizer implements HiveAuthorizer {
    public AbstractHiveAuthorizer() {
    }

    public HiveAuthorizationTranslator getHiveAuthorizationTranslator() throws HiveAuthzPluginException {
        return null;
    }

    public HivePolicyProvider getHivePolicyProvider() throws HiveAuthzPluginException {
        return null;
    }
}

RangerHiveAuthorizerBase

RangerHiveAuthorizerBase 实现了 3 个方法，更主要的是把构造参数存储在变量里，并提供公有的方法来获取。

public abstract class RangerHiveAuthorizerBase extends AbstractHiveAuthorizer {

	private static final Log LOG = LogFactory.getLog(RangerHiveAuthorizerBase.class);

	private HiveMetastoreClientFactory mMetastoreClientFactory;
	private HiveConf                   mHiveConf;
	private HiveAuthenticationProvider mHiveAuthenticator;
	private HiveAuthzSessionContext    mSessionContext;
	private UserGroupInformation       mUgi;

	public RangerHiveAuthorizerBase(HiveMetastoreClientFactory metastoreClientFactory,
									  HiveConf                   hiveConf,
									  HiveAuthenticationProvider hiveAuthenticator,
									  HiveAuthzSessionContext    context) {
		mMetastoreClientFactory = metastoreClientFactory;
		mHiveConf               = hiveConf;
		mHiveAuthenticator      = hiveAuthenticator;
		mSessionContext         = context;

		String userName = mHiveAuthenticator == null ? null : mHiveAuthenticator.getUserName();

		mUgi = userName == null ? null : UserGroupInformation.createRemoteUser(userName);

		if(mHiveAuthenticator == null) {
			LOG.warn("RangerHiveAuthorizerBase.RangerHiveAuthorizerBase(): hiveAuthenticator is null");
		} else if(StringUtil.isEmpty(userName)) {
			LOG.warn("RangerHiveAuthorizerBase.RangerHiveAuthorizerBase(): hiveAuthenticator.getUserName() returned null/empty");
		} else if(mUgi == null) {
			LOG.warn(String.format("RangerHiveAuthorizerBase.RangerHiveAuthorizerBase(): UserGroupInformation.createRemoteUser(%s) returned null", userName));
		}
	}

	public HiveMetastoreClientFactory getMetastoreClientFactory() {
		return mMetastoreClientFactory;
	}

	public HiveConf getHiveConf() {
		return mHiveConf;
	}

	public HiveAuthenticationProvider getHiveAuthenticator() {
		return mHiveAuthenticator;
	}

	public HiveAuthzSessionContext getHiveAuthzSessionContext() {
		return mSessionContext;
	}

	public UserGroupInformation getCurrentUserGroupInfo() {
		return mUgi;
	}

	@Override
	public void applyAuthorizationConfigPolicy(HiveConf hiveConf) throws HiveAuthzPluginException {
		LOG.debug("RangerHiveAuthorizerBase.applyAuthorizationConfigPolicy()");

		// from SQLStdHiveAccessController.applyAuthorizationConfigPolicy()
		if (mSessionContext != null && mSessionContext.getClientType() == CLIENT_TYPE.HIVESERVER2) {
			// Configure PREEXECHOOKS with DisallowTransformHook to disallow transform queries
			String hooks = hiveConf.getVar(ConfVars.PREEXECHOOKS).trim();
			if (hooks.isEmpty()) {
				hooks = DisallowTransformHook.class.getName();
			} else {
				hooks = hooks + "," + DisallowTransformHook.class.getName();
			}

			hiveConf.setVar(ConfVars.PREEXECHOOKS, hooks);

			SettableConfigUpdater.setHiveConfWhiteList(hiveConf);
		}
	}

	/**
	 * Show privileges for given principal on given object
	 * @param principal
	 * @param privObj
	 * @return
	 * @throws HiveAuthzPluginException
	 * @throws HiveAccessControlException
	 */
	@Override
	public List<HivePrivilegeInfo> showPrivileges(HivePrincipal principal, HivePrivilegeObject privObj)
			throws HiveAuthzPluginException, HiveAccessControlException {
		LOG.debug("RangerHiveAuthorizerBase.showPrivileges()");

		throwNotImplementedException("showPrivileges");

		return null;
	}

	@Override
	public VERSION getVersion() {
		return VERSION.V1;
	}

	private void throwNotImplementedException(String method) throws HiveAuthzPluginException {
		throw new HiveAuthzPluginException(method + "() not implemented in Ranger AbstractHiveAuthorizer");
	}

	@Override
	public HivePolicyProvider getHivePolicyProvider() throws HiveAuthzPluginException {
	    // TODO Auto-generated method stub
	    return null;
	}

}

RangerHiveAuthorizer

RangerHiveAuthorizer 创建一个静态的变量 RangerHivePlugin hivePlugin, 并且把请求转到 plugin 的相应方法处理，如 grantPrivileges 方法调用 hivePlugin.grantAccess(request, auditHandler);。

@Override
	public void grantPrivileges(List<HivePrincipal> hivePrincipals,
								List<HivePrivilege> hivePrivileges,
								HivePrivilegeObject hivePrivObject,
								HivePrincipal       grantorPrincipal,
								boolean             grantOption)
										throws HiveAuthzPluginException, HiveAccessControlException {
		if (LOG.isDebugEnabled()) {
				LOG.debug("grantPrivileges() => HivePrivilegeObject:" + toString(hivePrivObject, new StringBuilder()) + "grantorPrincipal: " + grantorPrincipal + "hivePrincipals" + hivePrincipals + "hivePrivileges" + hivePrivileges);
		}

		if(! RangerHivePlugin.UpdateXaPoliciesOnGrantRevoke) {
			throw new HiveAuthzPluginException("GRANT/REVOKE not supported in Ranger HiveAuthorizer. Please use Ranger Security Admin to setup access control.");
		}

		RangerHiveAuditHandler auditHandler = new RangerHiveAuditHandler();

		try {
			List<HivePrivilegeObject> outputs = new ArrayList<>(Arrays.asList(hivePrivObject));
			RangerHiveResource resource = getHiveResource(HiveOperationType.GRANT_PRIVILEGE, hivePrivObject, null, outputs);
			GrantRevokeRequest request  = createGrantRevokeData(resource, hivePrincipals, hivePrivileges, grantorPrincipal, grantOption);

			LOG.info("grantPrivileges(): " + request);
			if(LOG.isDebugEnabled()) {
				LOG.debug("grantPrivileges(): " + request);
			}

			hivePlugin.grantAccess(request, auditHandler);
		} catch(Exception excp) {
			throw new HiveAccessControlException(excp);
		} finally {
			auditHandler.flushAudit();
		}
	}

RangerHivePlugin

RangerHivePlugin 的初始化参数 appType，代表那个 app，如 HiveServer2 的 appType 为 “hiveServer2”。RangerBasePlugin 继承 RangerBasePlugin

public RangerHivePlugin(String appType) {
	super("hive", appType);
}

RangerBasePlugin

RangerBasePlugin 所有种类的 plugin 都继承 RangerBasePlugin，如 RangerHdfsPlugin,RangerHivePlugin

RangerBasePlugin 的构造方法，如 serviceType = “hive”, appId = “hiveServer2”

public RangerBasePlugin(String serviceType, String appId) {
		this(new RangerPluginConfig(serviceType, null, appId, null, null, null));
	}

RangerPluginConfig

serviceType = “hive”, appId = “hiveServer2”

public RangerPluginConfig(String serviceType, String serviceName, String appId, String clusterName, String clusterType, RangerPolicyEngineOptions policyEngineOptions) {
        super();

        addResourcesForServiceType(serviceType);

        addResourcesForServiceName(this.serviceType, this.serviceName);

        
        if (policyEngineOptions == null) {
            policyEngineOptions = new RangerPolicyEngineOptions();

            policyEngineOptions.configureForPlugin(this, propertyPrefix);
        }

        this.policyEngineOptions = policyEngineOptions;

        LOG.info(policyEngineOptions);
    }

addResourcesForServiceType， serviceType = “hive”

private void addResourcesForServiceType(String serviceType) {
        String auditCfg    = "ranger-" + serviceType + "-audit.xml";
        String securityCfg = "ranger-" + serviceType + "-security.xml";
        String sslCfg 	   = "ranger-" + serviceType + "-policymgr-ssl.xml";

        if (!addResourceIfReadable(auditCfg)) {
            addAuditResource(serviceType);
        }

        if (!addResourceIfReadable(securityCfg)) {
            addSecurityResource(serviceType);
        }

        if (!addResourceIfReadable(sslCfg)) {
            addSslConfigResource(serviceType);
        }
    }

addResourcesForServiceName

serviceType = “hive”, serviceName = null，所以不加载任何文件。

private void addResourcesForServiceName(String serviceType, String serviceName) {
        if (StringUtils.isNotBlank(serviceType) && StringUtils.isNotBlank(serviceName)) {
            String serviceAuditCfg    = "ranger-" + serviceType + "-" + serviceName + "-audit.xml";
            String serviceSecurityCfg = "ranger-" + serviceType + "-" + serviceName + "-security.xml";
            String serviceSslCfg      = "ranger-" + serviceType + "-" + serviceName + "-policymgr-ssl.xml";

            addResourceIfReadable(serviceAuditCfg);
            addResourceIfReadable(serviceSecurityCfg);
            addResourceIfReadable(serviceSslCfg);
        }
    }

RangerBasePlugin 的构造方法

public RangerBasePlugin(RangerPluginConfig pluginConfig) {
		this.pluginConfig  = pluginConfig;
		this.pluginContext = new RangerPluginContext(pluginConfig);

		Set<String> superUsers         = toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() + ".super.users"));
		Set<String> superGroups        = toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() + ".super.groups"));
		Set<String> auditExcludeUsers  = toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() + ".audit.exclude.users"));
		Set<String> auditExcludeGroups = toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() + ".audit.exclude.groups"));
		Set<String> auditExcludeRoles  = toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() + ".audit.exclude.roles"));
		Set<String> serviceAdmins      = toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() + ".service.admins"));

		setSuperUsersAndGroups(superUsers, superGroups);
		setAuditExcludedUsersGroupsRoles(auditExcludeUsers, auditExcludeGroups, auditExcludeRoles);
		setIsFallbackSupported(pluginConfig.getBoolean(pluginConfig.getPropertyPrefix() + ".is.fallback.supported", false));
		setServiceAdmins(serviceAdmins);

		RangerScriptExecutionContext.init(pluginConfig);

		this.chainedPlugins = initChainedPlugins();
	}

## RangerBasePlugin#init

RangerBasePlugin 的 init 里构建PolicyRefresher，PolicyRefresher 是一个线程，

public void init() {
		cleanup();

		AuditProviderFactory providerFactory = AuditProviderFactory.getInstance();

		if (!providerFactory.isInitDone()) {
			if (pluginConfig.getProperties() != null) {
				providerFactory.init(pluginConfig.getProperties(), getAppId());
			} else {
				LOG.error("Audit subsystem is not initialized correctly. Please check audit configuration. ");
				LOG.error("No authorization audits will be generated. ");
			}
		}

		if (!pluginConfig.getPolicyEngineOptions().disablePolicyRefresher) {
			refresher = new PolicyRefresher(this);
			LOG.info("Created PolicyRefresher Thread(" + refresher.getName() + ")");
			refresher.setDaemon(true);
			refresher.startRefresher();
		}

		for (RangerChainedPlugin chainedPlugin : chainedPlugins) {
			chainedPlugin.init();
		}
	}

RangerBasePlugin#grantAccess

public void grantAccess(GrantRevokeRequest request, RangerAccessResultProcessor resultProcessor) throws Exception {
		if(LOG.isDebugEnabled()) {
			LOG.debug("==> RangerBasePlugin.grantAccess(" + request + ")");
		}

		boolean isSuccess = false;

		try {
			RangerPolicyEngine policyEngine = this.policyEngine;

			if (policyEngine != null) {
				request.setZoneName(policyEngine.getUniquelyMatchedZoneName(request));
			}

			getAdminClient().grantAccess(request);

			isSuccess = true;
		} finally {
			auditGrantRevoke(request, "grant", isSuccess, resultProcessor);
		}

		if(LOG.isDebugEnabled()) {
			LOG.debug("<== RangerBasePlugin.grantAccess(" + request + ")");
		}
	}

PolicyRefresher 的构造方法

public PolicyRefresher(RangerBasePlugin plugIn) {

		RangerPluginConfig pluginConfig   = plugIn.getConfig();
		String             propertyPrefix = pluginConfig.getPropertyPrefix();

		this.plugIn      = plugIn;
		this.serviceType = plugIn.getServiceType();
		this.serviceName = plugIn.getServiceName();
		this.cacheDir    = pluginConfig.get(propertyPrefix + ".policy.cache.dir");

		String appId         = StringUtils.isEmpty(plugIn.getAppId()) ? serviceType : plugIn.getAppId();
		String cacheFilename = String.format("%s_%s.json", appId, serviceName);

		cacheFilename = cacheFilename.replace(File.separatorChar,  '_');
		cacheFilename = cacheFilename.replace(File.pathSeparatorChar,  '_');

		this.cacheFileName = cacheFilename;

		Gson gson = null;
		try {
			gson = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").create();
		} catch(Throwable excp) {
			LOG.fatal("PolicyRefresher(): failed to create GsonBuilder object", excp);
		}

		RangerPluginContext pluginContext  = plugIn.getPluginContext();
		RangerAdminClient   adminClient    = pluginContext.getAdminClient();
		this.rangerAdmin                   = (adminClient != null) ? adminClient : pluginContext.createAdminClient(pluginConfig);
		this.gson                          = gson;
		this.rolesProvider                 = new RangerRolesProvider(getServiceType(), appId, getServiceName(), rangerAdmin,  cacheDir, pluginConfig);
		this.pollingIntervalMs             = pluginConfig.getLong(propertyPrefix + ".policy.pollIntervalMs", 30 * 1000);

		setName("PolicyRefresher(serviceName=" + serviceName + ")-" + getId());
	}

特别注意

RangerAdminClient   adminClient    = pluginContext.getAdminClient();
		this.rangerAdmin                   = (adminClient != null) ? adminClient : pluginContext.createAdminClient(pluginConfig);

RangerPluginContext#createAdminClient

使用反射的方法创建 adminClient

public RangerAdminClient createAdminClient(RangerPluginConfig pluginConfig) {

		RangerAdminClient ret              = null;
		String            propertyName     = pluginConfig.getPropertyPrefix() + ".policy.source.impl";
		String            policySourceImpl = pluginConfig.get(propertyName);

		try {
			@SuppressWarnings("unchecked")
			Class<RangerAdminClient> adminClass = (Class<RangerAdminClient>)Class.forName(policySourceImpl);

			ret = adminClass.newInstance();
		} catch (Exception excp) {
			LOG.error("failed to instantiate policy source of type '" + policySourceImpl + "'. Will use policy source of type '" + RangerAdminRESTClient.class.getName() + "'", excp);
		}
		

		if(ret == null) {
			ret = new RangerAdminRESTClient();
		}

		ret.init(pluginConfig.getServiceName(), pluginConfig.getAppId(), pluginConfig.getPropertyPrefix(), pluginConfig);

		if(LOG.isDebugEnabled()) {
			LOG.debug("<== RangerBasePlugin.createAdminClient(" + pluginConfig.getServiceName() + ", " + pluginConfig.getAppId() + ", " + pluginConfig.getPropertyPrefix() + "): policySourceImpl=" + policySourceImpl + ", client=" + ret);
		}

		setAdminClient(ret);

		return ret;
	}

ranger-hive-security.xml

<configuration  xmlns:xi="http://www.w3.org/2001/XInclude">
    
    <property>
      <name>ranger.plugin.hive.policy.cache.dir</name>
      <value>/etc/ranger/${CLUSTER_ID}_hive/policycache</value>
    </property>
    
    <property>
      <name>ranger.plugin.hive.policy.pollIntervalMs</name>
      <value>30000</value>
    </property>
    
    <property>
      <name>ranger.plugin.hive.policy.rest.ssl.config.file</name>
      <value>/etc/hive/conf/ranger-policymgr-ssl.xml</value>
    </property>
    
    <property>
      <name>ranger.plugin.hive.policy.rest.url</name>
      <value>http://ranger-master:6080</value>
    </property>
    
    <property>
      <name>ranger.plugin.hive.policy.source.impl</name>
      <value>org.apache.ranger.admin.client.RangerAdminRESTClient</value>
    </property>
    
    <property>
      <name>ranger.plugin.hive.service.name</name>
      <value>${CLUSTER_ID}_hive</value>
    </property>
    
    <property>
      <name>ranger.plugin.hive.urlauth.filesystem.schemes</name>
      <value>hdfs:,file:,wasb:,adl:</value>
    </property>
    
    <property>
      <name>xasecure.hive.update.xapolicies.on.grant.revoke</name>
      <value>true</value>
    </property>

可以看到，配置的是 RangerAdminRESTClient。

RangerAdminRESTClient#init

@Override
	public void init(String serviceName, String appId, String propertyPrefix, Configuration config) {
	    super.init(serviceName, appId, propertyPrefix, config);

		this.serviceName = serviceName;
		this.pluginId    = restUtils.getPluginId(serviceName, appId);

		String url                      = "";
		String tmpUrl                   = config.get(propertyPrefix + ".policy.rest.url");
		String sslConfigFileName 		= config.get(propertyPrefix + ".policy.rest.ssl.config.file");
		clusterName       				= config.get(propertyPrefix + ".access.cluster.name", "");
		if(StringUtil.isEmpty(clusterName)){
			clusterName =config.get(propertyPrefix + ".ambari.cluster.name", "");
			if (StringUtil.isEmpty(clusterName)) {
				if (config instanceof RangerPluginConfig) {
					clusterName = ((RangerPluginConfig)config).getClusterName();
				}
			}
		}
		int	 restClientConnTimeOutMs	= config.getInt(propertyPrefix + ".policy.rest.client.connection.timeoutMs", 120 * 1000);
		int	 restClientReadTimeOutMs	= config.getInt(propertyPrefix + ".policy.rest.client.read.timeoutMs", 30 * 1000);
		supportsPolicyDeltas            = config.getBoolean(propertyPrefix + RangerCommonConstants.PLUGIN_CONFIG_SUFFIX_POLICY_DELTA, RangerCommonConstants.PLUGIN_CONFIG_SUFFIX_POLICY_DELTA_DEFAULT);
		supportsTagDeltas               = config.getBoolean(propertyPrefix + RangerCommonConstants.PLUGIN_CONFIG_SUFFIX_TAG_DELTA, RangerCommonConstants.PLUGIN_CONFIG_SUFFIX_TAG_DELTA_DEFAULT);
		isRangerCookieEnabled			= config.getBoolean(propertyPrefix + ".policy.rest.client.cookie.enabled", RangerCommonConstants.POLICY_REST_CLIENT_SESSION_COOKIE_ENABLED);
		rangerAdminCookieName			= config.get(propertyPrefix + ".policy.rest.client.session.cookie.name", RangerCommonConstants.DEFAULT_COOKIE_NAME);

        if (!StringUtil.isEmpty(tmpUrl)) {
            url = tmpUrl.trim();
        }
        if (url.endsWith("/")) {
            url = url.substring(0, url.length() - 1);
        }

		init(url, sslConfigFileName, restClientConnTimeOutMs , restClientReadTimeOutMs, config);

        try {
            this.serviceNameUrlParam = URLEncoderUtil.encodeURIParam(serviceName);
        } catch (UnsupportedEncodingException e) {
            LOG.warn("Unsupported encoding, serviceName=" + serviceName);
            this.serviceNameUrlParam = serviceName;
        }
	}

RangerAdminRESTClient#init

在此方法里，创建了和 Ranger Server 的 HTTP 连接。

private void init(String url, String sslConfigFileName, int restClientConnTimeOutMs , int restClientReadTimeOutMs, Configuration config) {
		if(LOG.isDebugEnabled()) {
			LOG.debug("==> RangerAdminRESTClient.init(" + url + ", " + sslConfigFileName + ")");
		}

		restClient = new RangerRESTClient(url, sslConfigFileName, config);
		restClient.setRestClientConnTimeOutMs(restClientConnTimeOutMs);
		restClient.setRestClientReadTimeOutMs(restClientReadTimeOutMs);

		if(LOG.isDebugEnabled()) {
			LOG.debug("<== RangerAdminRESTClient.init(" + url + ", " + sslConfigFileName + ")");
		}
	}

PolicyRefresher#run 方法

先从 policyDownloadQueue 去出一个元素，执行一次 loadRoles() 和 loadPolicy()

public void run() {

		if(LOG.isDebugEnabled()) {
			LOG.debug("==> PolicyRefresher(serviceName=" + serviceName + ").run()");
		}

		while(true) {
			DownloadTrigger trigger = null;
			try {
				trigger = policyDownloadQueue.take();
				loadRoles();
				loadPolicy();
			} catch(InterruptedException excp) {
				LOG.info("PolicyRefresher(serviceName=" + serviceName + ").run(): interrupted! Exiting thread", excp);
				break;
			} finally {
				if (trigger != null) {
					trigger.signalCompletion();
				}
			}
		}

		if(LOG.isDebugEnabled()) {
			LOG.debug("<== PolicyRefresher(serviceName=" + serviceName + ").run()");
		}
	}

PolicyRefresher#startRefresher

在此方法里，另启动一个 Timer， 定期（默认 30 秒）向 policyDownloadQueue 里放一个 Object。也代表 PolicyRefresher 线程定期执行一次 loadRoles() 和 loadPolicy()。

public void startRefresher() {
		loadRoles();
		loadPolicy();

		super.start();

		policyDownloadTimer = new Timer("policyDownloadTimer", true);

		try {
			policyDownloadTimer.schedule(new DownloaderTask(policyDownloadQueue), pollingIntervalMs, pollingIntervalMs);

			if (LOG.isDebugEnabled()) {
				LOG.debug("Scheduled policyDownloadRefresher to download policies every " + pollingIntervalMs + " milliseconds");
			}
		} catch (IllegalStateException exception) {
			LOG.error("Error scheduling policyDownloadTimer:", exception);
			LOG.error("*** Policies will NOT be downloaded every " + pollingIntervalMs + " milliseconds ***");

			policyDownloadTimer = null;
		}

	}

PolicyRefresher#loadRoles

plugIn 是

private void loadRoles() {
		if(LOG.isDebugEnabled()) {
			LOG.debug("==> PolicyRefresher(serviceName=" + serviceName + ").loadRoles()");
		}

		//Load the Ranger UserGroup Roles
		rolesProvider.loadUserGroupRoles(plugIn);

		if(LOG.isDebugEnabled()) {
			LOG.debug("<== PolicyRefresher(serviceName=" + serviceName + ").loadRoles()");
		}
	}

PolicyRefresher#loadUserGroupRoles

public void loadUserGroupRoles(RangerBasePlugin plugIn) {
	RangerRoles roles = loadUserGroupRolesFromAdmin();

   plugIn.setRoles(null);
}

RangerRolesProvider#loadUserGroupRolesFromAdmin

private RangerRoles loadUserGroupRolesFromAdmin() throws RangerServiceNotFoundException {

			roles = rangerAdmin.getRolesIfUpdated(lastKnownRoleVersion, lastActivationTimeInMillis);
		 return roles;
	}
houzhizhen
关注
  • 1
    点赞
  • 踩
  • 2
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
专栏目录
APACHE RANGER 调研----ranger 原理解析
01-26
(1) 管理员设置策略以及用户(例如一个用户对一个hive数据库相关的权限）
ranger-3.0.0-SNAPSHOT-hive-plugin.tar.gz
12-18
ranger-3.0.0-SNAPSHOT-hive-plugin.tar.gz
ranger-2.0.0-hive-plugin.tar.gz
06-04
由于很多文章都是只有ranger的安装说明，却没有编译后的tar包，而且编译的时间有点长，编译期间会出现各种问题，导致编译不过，特此上传编译后的包，来提供大家下载
ranger-2.1.0-hive-plugin.tar.gz
08-09
ranger-2.1.0-hive-plugin.tar.gz
ranger-2.1.0-kafka-plugin.tar.gz
08-09
ranger-2.1.0-kafka-plugin.tar.gz
hive插件 ranger_阅读ranger-hive-plugin源码
weixin_39933414的博客
12-22 622
首先ranger和hive是通过在hive中配置hive.security.authorization.manager参数，建立起的联系在hive的lib目录下存在ranger-plugin-classloader-0.5.0.2.4.0.0-169.jar、ranger-hive-plugin-shim-0.5.0.2.4.0.0-169.jar这两jar包以及ranger-hive-plugi...
Ranger 0.5.4-SNAPSHOT CDH 5.8.4 HDFS Plugin Hive Plugin
huangyq5240的博客
10-24 1119
组件版本信息： Hadoop 版本：cdh-5.8.4 Hive 版本: 1.1.0-cdh5.8.4 Ranger 版本: ranger-0.5.4-SNAPSHOT Solr 版本: 5.2.1 Jdk 版本：1.7.80 前提条件： cdh-5.8.4 需要修复 HDFS-6826，否则Ranger HDFS Plugin编译无法通过 本地mvn仓库编译安装修改后hdfs依赖包： [cl...
Ranger (四) --------- 安装 Ranger Hive-plugin
在森林里麋了鹿
11-03 1352
安装 Ranger Hive-plugin
Ranger集成hive 安装与测试
wutiantesi的博客
06-17 3005
Ranger集成hive 安装与测试前期准备软件安装下载ranger2.0.0解压编译源文件解压安装安装Ranger控制台：Ranger—admin安装Ranger UsersyncHive插件安装创建Hive Service配置权限Row Level Filter(行级别过滤)参考博客声明 前期准备 ranger2.0.0 hive2.3.7 hive数据库创建t_user表 CREATE TABLE t_user (name STRING,phone STRING) ROW FORMAT D
Apache Ranger：(二)对Hive集成简单使用
最新发布
小健的博客
10-09 539
进入 Ranger 编译生成的目录下 找到 ranger-2.0.0-hive-plugin.tar.gz 进行解压。
ranger-2.2.0-plugin.zip
04-22
ranger-2.2.0-plugin编译集合包，其他包可搜ranger-2.2.0相关信息
Ranger集成CDH Hive
qq_35741557的博客
05-28 1877
安装 HIVE插件只对使用JDBC方式访问HIVE的请求进行权限管理，HIVE-CLI不受限制！ 解压文件ranger-2.1.0-hive-plugin.tar.gz并修改install.properties文件 #ranger admin的路径 POLICY_MGR_URL=http://cdh04:6080 #hive服务名称 REPOSITORY_NAME=hivedev #hive的安装路径 COMPONENT_INSTALL_DIR_NAME=/opt/cloudera/parcels/CDH/
Hive(九) Hive 权限
奥迪的博客
09-24 1094
Hive 权限 https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Authorization Introduction Note that this documentation is referring to Authorization which is verifying if a user has permission to perform a certain action, and not about Authenti
干货：一文看懂Apache Ranger | 凌云时刻
云布道师
05-26 5952
凌云时刻 ·技术导读：Apache Ranger来源于XA Secure公司。2013年，XA Secure在加利福尼亚成立，专门做Hadoop生态的安全管控。2014年，Horton...
Hive：权限管理
程序猿启程
08-09 1088
Storage Based Authorization in the Metastore Server 基于存储的授权，可以对 Metastore 中的元数据进行保护，但是没有提供更加细粒度的访问控制（如：列级别、行级别） SQL Standards Based Authorization in HiveServer2 ​​​​​​​基于 SQL 标准的 Hive 授权，完全兼容 SQL 的授权模型，推荐使用该模式 Default Hive Authorization（Legacy Mode
ranger2.1.0源码编译以及安装
qq_39381847的博客
07-24 500
【代码】ranger2.1.0源码编译以及安装。
Apache Ranger源码编译及使用
热门推荐
数据科学汇集
12-28 1万+
Ranger源码编译、使用手册 1 Ranger简介 Apache Ranger提供一个集中式安全管理框架,它可以对Hadoop生态的组件如Hive,Hbase进行细粒度的数据访问控制.通过操作Ranger控制台,管理员可以轻松的通过配置策略来控制用户访问HDFS文件夹、HDFS文件、数据库、表、字段权限.这些策略可以为不同的用户和组来设置,同时权限可与hadoop无缝对接. 2 准备 2
HIVE Statistics(Hive 统计信息) 的说明和查询优化详解
houzhizhen的专栏
03-09 7741
本文从普通用户的角度讲述 Hive 统计信息，源代码角度请参考。 统计信息的分类 有两种统计信息，第1种为表和分区的统计信息，第2种为分区的统计信息。 表和分区的统计信息 表的统计信息 包括如下内容： numFiles – 文件的数量 numRows – 记录数量 totalSize – 总文件大小 rawDataSize – 原始数据量（因为可以压缩后存储到文件里） 对于分区表，还包括以下内容： numPartitions 1824 显示表的统计信息的语句。 des
Hive container 内存溢出的解决方法
houzhizhen的专栏
12-18 6918
调大 container 内存 container 内存溢出，可以使用set hive.tez.container.size=8192; 或者 set hive.tez.container.size=16384; 调大container 的内存。 增大 reduce 的数量,减少每个 reduce 处理的数据量 执行 set hive.exec.reducers.max; 可以查看当前系统最大 reduce 的数量，如果任务的 reduce task 数量已经达到最大值。设置 set hive.exec.
作为一个资深的python算法工程师，你需要用python实现对hivesql血缘解析，可解析所有hive高级语法，输出源库表，目标库表，以及源字段和目标字段一一对应可使用sqllineage ，遇到*字段可连接hive源实例解析，输出一套完整的python代码
06-09
好的，这是一个非常有挑战性的问题。我会尽我所能来回答它。 首先，我需要说明一下，血缘解析是一个非常复杂的任务，需要深入理解Hive的语法和元数据，同时还需要处理诸如动态分区、UDF等高级特性。因此，我建议你可以考虑使用已有的开源工具来实现这个功能，比如Apache Atlas、Apache Ranger等等。 如果你决定自己实现，那么下面是一个可能的思路： 1. 解析Hive SQL语句：你可以使用Python中的ANTLR4库来解析Hive SQL语句。ANTLR4是一个非常强大的解析器生成器，可以根据你提供的语法规则生成解析器。 2. 提取元数据信息：在解析出SQL语句之后，你需要提取元数据信息，比如源表名、目标表名、字段名等等。你可以使用Python中的PyHive库来访问Hive的元数据信息。 3. 解析*字段：当你遇到*字段时，你需要连接Hive源实例来解析该表的所有字段。你可以使用Python中的PyHive库来连接Hive源实例，并使用SHOW COLUMNS语句来获取该表的所有字段信息。 4. 输出代码：最后，你需要将解析出来的信息输出为一套完整的Python代码，以便你或其他人可以使用。 下面是一个简单的示例代码，它可以解析出一个Hive SQL语句中的源表名、目标表名、源字段和目标字段一一对应的关系： ```python from antlr4 import * from antlr4.tree.Trees import Trees from antlr4.error.ErrorListener import ErrorListener from pyhive import hive # 自定义错误监听器 class MyErrorListener(ErrorListener): def syntaxError(self, recognizer, offendingSymbol, line, column, msg, e): raise Exception("line {}:{} {}".format(line, column, msg)) # 解析Hive SQL语句 def parse_hive_sql(sql): from HiveLexer import HiveLexer from HiveParser import HiveParser input_stream = InputStream(sql) lexer = HiveLexer(input_stream) stream = CommonTokenStream(lexer) parser = HiveParser(stream) parser.removeErrorListeners() error_listener = MyErrorListener() parser.addErrorListener(error_listener) tree = parser.singleStatement() return tree # 提取源表名、目标表名、源字段和目标字段 def extract_metadata(tree): source_table_name = '' target_table_name = '' source_fields = [] target_fields = [] for child in Trees.getChildren(tree): if 'fromSource' in child.__class__.__name__: source_table_name = child.sourceName().getText() for column in child.columnAliases().children: if column.getText() != ',': source_fields.append(column.getText()) elif 'intoTarget' in child.__class__.__name__: target_table_name = child.targetName().getText() for column in child.columnAliases().children: if column.getText() != ',': target_fields.append(column.getText()) return source_table_name, target_table_name, source_fields, target_fields # 解析*字段 def parse_star_fields(fields, connection): parsed_fields = [] if '*' in fields: table_name = fields.split('.')[0] cursor = connection.cursor() cursor.execute('SHOW COLUMNS IN {}'.format(table_name)) result = cursor.fetchall() for row in result: parsed_fields.append(row[0]) else: parsed_fields = fields.split(',') return parsed_fields # 输出代码 def output_code(source_table_name, target_table_name, source_fields, target_fields): print('source_table_name = "{}"'.format(source_table_name)) print('target_table_name = "{}"'.format(target_table_name)) print('source_fields = {}'.format(source_fields)) print('target_fields = {}'.format(target_fields)) # 主函数 def main(sql): tree = parse_hive_sql(sql) source_table_name, target_table_name, source_fields, target_fields = extract_metadata(tree) connection = hive.Connection(host='localhost', port=10000, username='username', password='password', database='default') source_fields = parse_star_fields(source_fields, connection) target_fields = parse_star_fields(target_fields, connection) output_code(source_table_name, target_table_name, source_fields, target_fields) if __name__ == '__main__': sql = 'INSERT INTO target_table SELECT * FROM source_table' main(sql) ``` 需要注意的是，这只是一个非常简单的示例代码，实际上实现一个完整的血缘解析器要比这个复杂得多。我建议你可以参考一些已有的开源工具，来更好地理解如何实现这个功能。

“相关推荐”对你有帮助么？

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值