# 需要导入模块: import httplib2 [as 别名]
# 或者: from httplib2 import debuglevel [as 别名]
def test_dataproc_resource():
'''Tests dataproc cluster creation/deletion. Requests are captured by the responses library, so
no actual HTTP requests are made here.
Note that inspecting the HTTP requests can be useful for debugging, which can be done by adding:
import httplib2
httplib2.debuglevel = 4
'''
with mock.patch('httplib2.Http', new=HttpSnooper):
pipeline = PipelineDefinition(
name='test_dataproc_resource',
solid_defs=[dataproc_solid],
mode_defs=[ModeDefinition(resource_defs={'dataproc': dataproc_resource})],
)
result = execute_pipeline(
pipeline,
{
'solids': {
'dataproc_solid': {
'config': {
'job_config': {
'projectId': PROJECT_ID,
'region': REGION,
'job': {
'reference': {'projectId': PROJECT_ID},
'placement': {'clusterName': CLUSTER_NAME},
'hiveJob': {'queryList': {'queries': ['SHOW DATABASES']}},
},
},
'job_scoped_cluster': True,
}
}
},
'resources': {
'dataproc': {
'config': {
'projectId': PROJECT_ID,
'clusterName': CLUSTER_NAME,
'region': REGION,
'cluster_config': {
'softwareConfig': {
'properties': {
# Create a single-node cluster
# This needs to be the string "true" when
# serialized, not a boolean true
'dataproc:dataproc.allow.zero.workers': 'true'
}
}
},
}
}
},
},
)
assert result.success