fluent-bit日志上传到s3-efs

在线 JSON 工具，JSON 校验/格式化/压缩/工具 - 在线工具-wetools.com微工具

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "elasticfilesystem:ClientMount",
                "elasticfilesystem:ClientWrite"
            ],
            "Resource": "arn:aws:elasticfilesystem:REGION:ACCOUNT_ID:file-system/fs-xxxxxx",
            "Condition": {
                "StringEquals": {
                    "elasticfilesystem:AccessPointArn": "arn:aws:elasticfilesystem:REGION:ACCOUNT_ID:access-point/fsap-xxxxxxxxxxxxx"
                }
            }
        }
    ]
}

s3存储桶设置权限：

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Principal": "*",
            "Sid": "AllowS3Self",
            "Effect": "Allow",
            "Action": "s3:*", 
            "Resource": [
                "arn:aws:s3:::firelens-access-log/*", 
                "arn:aws:s3:::firelens-access-log" 
            ]
        }
       
    ]
}
 

{
    "family": "firelens-sample-s3-ec2-demo",
    "taskRoleArn": "arn:aws:iam::254278701124:role/ecsTaskExecutionRole",
    "executionRoleArn": "arn:aws:iam::254278701124:role/ecsTaskExecutionRole",
    "containerDefinitions": [
        {
            "essential": true,
            "name": "log_router",
            "image": "402097323/fluent-bit-s3:v2",
            "logConfiguration": {
                "logDriver": "awslogs",
                "options": {
                    "awslogs-create-group": "true",
                    "awslogs-group": "/ecs/firelens-sample",
                    "awslogs-region": "us-east-1",
                    "awslogs-stream-prefix": "ecs"
                }
            },
            "mountPoints": [
                {
                    "sourceVolume": "data",
                    "containerPath": "/data",
                    "readOnly": false
                }
            ],
            "firelensConfiguration": {
                "type": "fluentbit",
                "options": {
                    "config-file-type": "file",
                    "config-file-value": "/extra.conf",
                    "enable-ecs-log-metadata": "false"
                }
            },
            "user": "0"
        },
        {
            "essential": true,
            "name": "myapp",
            "image": "402097323/applog:v2",
            "logConfiguration": {
                "logDriver": "awsfirelens"
            },
            "mountPoints": [
                {
                    "sourceVolume": "data",
                    "containerPath": "/data",
                    "readOnly": false
                }
            ],
            "dependsOn": [
                {
                    "containerName": "log_router",
                    "condition": "START"
                }
            ]
        }
    ],
    "cpu": "256",
    "memory": "512",
    "volumes": [
        {
            "name": "data",
            "host": {}
        }
    ],
    "compatibilities": [
    "EXTERNAL",
    "EC2"
  ],
     "networkMode": "awsvpc"
}

在老版本中设置：

{
    "family": "firelens-sample-s3-ec2-demo-cre",
    "taskRoleArn": "arn:aws:iam::254278701124:role/ecsTaskExecutionRole",
    "executionRoleArn": "arn:aws:iam::254278701124:role/ecsTaskExecutionRole",
    "containerDefinitions": [{
        "essential": true,
        "name": "log_router",
        "image": "402097323/fluent-bit-s3:v2",
        "logConfiguration": {
            "logDriver": "awslogs",
            "options": {
                "awslogs-create-group": "true",
                "awslogs-group": "/ecs/firelens-sample",
                "awslogs-region": "us-east-1",
                "awslogs-stream-prefix": "ecs"
            }
        },
        "mountPoints": [{
            "sourceVolume": "data",
            "containerPath": "/data",
            "readOnly": false
        }],
        "firelensConfiguration": {
            "type": "fluentbit",
            "options": {
                "config-file-type": "file",
                "config-file-value": "/extra.conf",
                "enable-ecs-log-metadata": "false"
            }
        },
        "user": "0"
    }, {
        "essential": true,
        "name": "myapp",
        "image": "402097323/applog:v2",
        "logConfiguration": {
            "logDriver": "awsfirelens"
        },
        "mountPoints": [{
            "sourceVolume": "data",
            "containerPath": "/data",
            "readOnly": false
        }],
        "dependsOn": [{
            "containerName": "log_router",
            "condition": "START"
        }]
    }],
    "cpu": "256",
    "memory": "512",
    "volumes": [{
        "name": "data",
        "host": {}
    }],
    "networkMode": "awsvpc"
}

/extra.conf根据实际情况监控路径，402097323/fluent-bit-s3:v2 这个镜像要启动容器，然后把extra.conf文件复制到根路径下

docker run -itd --name ffff1 402097323/fluent-bit-s3:v2

docker cp 75bb7ae358a1:/extra.conf   extra.conf

docker commit 容器实例id  镜像名称

网络部署成桥接模式，否则可能会网络不通，在加上s3权限设置；任务类型别设置fargate

[SERVICE]
    Parsers_File /fluent-bit/parsers/parsers.conf
    Flush 1
    Grace 30

[INPUT]
    Name tail
    Path /data/logs/access.log
    Tag access

[INPUT]
    Name tail
    Path /data/logs/error.log
    Tag error

[OUTPUT]
    Name s3
    Match *
    bucket firelens-access-log
    region us-east-1
    upload_timeout  1m
    total_file_size 10M

json数据要参看firelen配置  filter   部分，否则json中会有\错误

docker logs 088fc8999f35 -f

[2023/05/09 12:09:50] [ info] [output:s3:s3.0] Successfully uploaded object /fluent-bit-logs/access/2023/05/09/12/08/40-objectfDHCcciD
[2023/05/09 12:09:50] [ info] [output:s3:s3.0] Successfully uploaded object /fluent-bit-logs/error/2023/05/09/12/08/40-objectBwI4KFtG
[2023/05/09 12:11:00] [ info] [output:s3:s3.0] Successfully uploaded object /fluent-bit-logs/access/2023/05/09/12/09/50-objectMQrxDKaJ

sudo /opt/fluent-bit/bin/fluent-bit -i cpu -o s3 -p bucket=firelens-access-log -p region=us-east-1 -p -m '*' -f 1

sudo  bin/fluent-bit -i tail -p '/var/log/messages' -F grep -p 'regex=log aa' -m '*' -o stdout

Fluent Bit v2.1 Documentation - Fluent Bit: Official Manual