loki采集k8s日志
前言
loki 是轻量、易用的日志聚合系统。如果你的k8s集群规模并不大，推荐使用grafana+loki的方案来做微服务日志的采集；
Loki组成
loki架构很简单，主要由3部分组成：
loki：服务端，负责存储日志和处理查询；
promtail：采集端，负责采集日志发送给loki；
grafana：负责采集日志的展示；
环境：
k8s v1.22.10
创建loki资源
vim loki-rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:name: lokinamespace: thanos-monitoring---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:name: lokinamespace: thanos-monitoring
rules:
- apiGroups:- extensionsresourceNames:- lokiresources:- podsecuritypoliciesverbs:- use---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:name: lokinamespace: thanos-monitoring
roleRef:apiGroup: rbac.authorization.k8s.iokind: Rolename: loki
subjects:
- kind: ServiceAccountname: loki
vim loki-nodeport.yaml
apiVersion: v1
kind: Service
metadata:name: loki-nodeportnamespace: thanos-monitoring
spec:type: NodePortports:- name: httpport: 3100targetPort: 3100nodePort: 30100selector:app: loki
vim loki.yaml
---
apiVersion: v1
kind: Service
metadata:name: loki-headlessnamespace: thanos-monitoringlabels:app: loki
spec:type: ClusterIPclusterIP: Noneports:- port: 3100protocol: TCPname: http-metricstargetPort: http-metricsselector:app: loki---
apiVersion: v1
kind: Service
metadata:name: lokinamespace: thanos-monitoringlabels:app: loki
spec:type: ClusterIPports:- port: 3100protocol: TCPname: http-metricstargetPort: http-metricsselector:app: loki---
apiVersion: apps/v1
kind: StatefulSet
metadata:name: lokinamespace: thanos-monitoringlabels:app: loki
spec:podManagementPolicy: OrderedReadyreplicas: 1selector:matchLabels:app: lokiserviceName: loki-headlessupdateStrategy:type: RollingUpdatetemplate:metadata:labels:app: lokispec:serviceAccountName: lokisecurityContext:fsGroup: 10001runAsGroup: 10001runAsNonRoot: truerunAsUser: 10001initContainers: []containers:- name: lokiimage: grafana/loki:2.0.0imagePullPolicy: IfNotPresentargs:- -config.file=/etc/loki/loki.yamlvolumeMounts:- name: configmountPath: /etc/loki- name: storagemountPath: /dataports:- name: http-metricscontainerPort: 3100protocol: TCPlivenessProbe:httpGet: path: /readyport: http-metricsscheme: HTTPinitialDelaySeconds: 45timeoutSeconds: 1periodSeconds: 10successThreshold: 1failureThreshold: 3readinessProbe:httpGet: path: /readyport: http-metricsscheme: HTTPinitialDelaySeconds: 45timeoutSeconds: 1periodSeconds: 10successThreshold: 1failureThreshold: 3securityContext:readOnlyRootFilesystem: trueterminationGracePeriodSeconds: 4800volumes:- name: configconfigMap:defaultMode: 420name: lokivolumeClaimTemplates:- metadata:name: storagelabels:app: lokispec:storageClassName: managed-nfs-storageaccessModes:- ReadWriteManyresources:requests:storage: "2Gi"
vim loki-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:name: lokinamespace: thanos-monitoringlabels:app: loki
data:loki.yaml: |auth_enabled: falseingester:chunk_idle_period: 3m # 如果块没有达到最大的块大小，那么在刷新之前，块应该在内存中不更新多长时间chunk_block_size: 262144chunk_retain_period: 1m # 块刷新后应该在内存中保留多长时间max_transfer_retries: 0 # Number of times to try and transfer chunks when leaving before falling back to flushing to the store. Zero = no transfers are done.lifecycler: #配置ingester的生命周期，以及在哪里注册以进行发现ring:kvstore:store: inmemory # 用于ring的后端存储，支持consul、etcd、inmemoryreplication_factor: 1 # 写入和读取的ingesters数量，至少为1（为了冗余和弹性，默认情况下为3）limits_config:enforce_metric_name: falsereject_old_samples: true # 旧样品是否会被拒绝reject_old_samples_max_age: 168h # 拒绝旧样本的最大时限schema_config: # 配置从特定时间段开始应该使用哪些索引模式configs:- from: 2023-04-12 # 创建索引的日期。如果这是唯一的schema_config，则使用过去的日期，否则使用希望切换模式时的日期store: boltdb-shipper # 索引使用哪个存储，如：cassandra, bigtable, dynamodb，或boltdbobject_store: filesystem # 用于块的存储，如：gcs, s3， inmemory, filesystem, cassandra，如果省略，默认值与store相同schema: v11index: # 配置如何更新和存储索引prefix: index_ # 所有周期表的前缀period: 24h # 表周期server:http_listen_port: 3100storage_config: # 为索引和块配置一个或多个存储boltdb_shipper:active_index_directory: /data/loki/boltdb-shipper-activecache_location: /data/loki/boltdb-shipper-cachecache_ttl: 24h shared_store: filesystemfilesystem:directory: /data/loki/chunkschunk_store_config: # 配置如何缓存块，以及在将它们保存到存储之前等待多长时间max_look_back_period: 0s #限制查询数据的时间，默认是禁用的，这个值应该小于或等于table_manager.retention_period中的值table_manager:retention_deletes_enabled: false # 日志保留周期开关，用于表保留删除retention_period: 0s # 日志保留周期，保留期必须是索引/块的倍数compactor:working_directory: /data/loki/boltdb-shipper-compactorshared_store: filesystem
标题创建promtail资源
vim loki-promtail-rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:name: loki-promtaillabels:app: promtailnamespace: thanos-monitoring---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:labels:app: promtailname: promtail-clusterrole
rules:
- apiGroups: [""] # "" indicates the core API groupresources:- nodes- nodes/proxy- services- endpoints- podsverbs: ["get", "watch", "list"]---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: promtail-clusterrolebindinglabels:app: promtail
subjects:- kind: ServiceAccountname: loki-promtailnamespace: thanos-monitoring
roleRef:kind: ClusterRolename: promtail-clusterroleapiGroup: rbac.authorization.k8s.io
vim loki-promtail.yaml
apiVersion: v1
kind: Service
metadata:name: loki-promtail-headlessnamespace: thanos-monitoringlabels:app: promtail
spec:clusterIP: Noneports:- port: 3101protocol: TCPname: http-metricstargetPort: http-metricsselector:app: promtail---
apiVersion: apps/v1
kind: DaemonSet
metadata:name: loki-promtailnamespace: thanos-monitoringlabels:app: promtail
spec:selector:matchLabels:app: promtailupdateStrategy:rollingUpdate:maxUnavailable: 1type: RollingUpdatetemplate:metadata:labels:app: promtailspec:serviceAccountName: loki-promtailcontainers:- name: promtailimage: grafana/promtail:2.0.0imagePullPolicy: IfNotPresentargs: - -config.file=/etc/promtail/promtail.yaml- -client.url=http://loki:3100/loki/api/v1/pushenv: - name: HOSTNAMEvalueFrom: fieldRef: apiVersion: v1fieldPath: spec.nodeNamevolumeMounts:- mountPath: /etc/promtailname: config- mountPath: /run/promtailname: run- mountPath: /var/lib/docker/containersname: dockerreadOnly: true- mountPath: /var/log/podsname: podsreadOnly: trueports:- containerPort: 3101name: http-metricsprotocol: TCPsecurityContext:readOnlyRootFilesystem: truerunAsGroup: 0runAsUser: 0readinessProbe:failureThreshold: 5httpGet:path: /readyport: http-metricsscheme: HTTPinitialDelaySeconds: 10periodSeconds: 10successThreshold: 1timeoutSeconds: 1tolerations:- effect: NoSchedulekey: node-role.kubernetes.io/masteroperator: Existsvolumes:- name: configconfigMap:defaultMode: 420name: loki-promtail- name: runhostPath:path: /run/promtailtype: ""- name: dockerhostPath:path: /var/lib/docker/containers- name: podshostPath:path: /var/log/pods
vim loki-promtail-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:name: loki-promtailnamespace: thanos-monitoringlabels:app: promtail
data:promtail.yaml: |client: # 配置Promtail如何连接到Loki的实例backoff_config: # 配置当请求失败时如何重试请求给Lokimax_period: 5m max_retries: 10min_period: 500msbatchsize: 1048576 # 发送给Loki的最大批次大小(以字节为单位)batchwait: 1s # 发送批处理前等待的最大时间（即使批次大小未达到最大值）external_labels: {} # 所有发送给Loki的日志添加静态标签timeout: 10s # 等待服务器响应请求的最大时间positions:filename: /run/promtail/positions.yamlserver:http_listen_port: 3101target_config:sync_period: 10sscrape_configs:- job_name: kubernetes-pods-namepipeline_stages:- docker: {}kubernetes_sd_configs:- role: podrelabel_configs:- source_labels:- __meta_kubernetes_pod_label_nametarget_label: __service__- source_labels:- __meta_kubernetes_pod_node_nametarget_label: __host__- action: dropregex: ''source_labels:- __service__- action: labelmapregex: __meta_kubernetes_pod_label_(.+)- action: replacereplacement: $1separator: /source_labels:- __meta_kubernetes_namespace- __service__target_label: job- action: replacesource_labels:- __meta_kubernetes_namespacetarget_label: namespace- action: replacesource_labels:- __meta_kubernetes_pod_nametarget_label: pod- action: replacesource_labels:- __meta_kubernetes_pod_container_nametarget_label: container- replacement: /var/log/pods/*$1/*.logseparator: /source_labels:- __meta_kubernetes_pod_uid- __meta_kubernetes_pod_container_nametarget_label: __path__- job_name: kubernetes-pods-apppipeline_stages:- docker: {}kubernetes_sd_configs:- role: podrelabel_configs:- action: dropregex: .+source_labels:- __meta_kubernetes_pod_label_name- source_labels:- __meta_kubernetes_pod_label_apptarget_label: __service__- source_labels:- __meta_kubernetes_pod_node_nametarget_label: __host__- action: dropregex: ''source_labels:- __service__- action: labelmapregex: __meta_kubernetes_pod_label_(.+)- action: replacereplacement: $1separator: /source_labels:- __meta_kubernetes_namespace- __service__target_label: job- action: replacesource_labels:- __meta_kubernetes_namespacetarget_label: namespace- action: replacesource_labels:- __meta_kubernetes_pod_nametarget_label: pod- action: replacesource_labels:- __meta_kubernetes_pod_container_nametarget_label: container- replacement: /var/log/pods/*$1/*.logseparator: /source_labels:- __meta_kubernetes_pod_uid- __meta_kubernetes_pod_container_nametarget_label: __path__- job_name: kubernetes-pods-direct-controllerspipeline_stages:- docker: {}kubernetes_sd_configs:- role: podrelabel_configs:- action: dropregex: .+separator: ''source_labels:- __meta_kubernetes_pod_label_name- __meta_kubernetes_pod_label_app- action: dropregex: '[0-9a-z-.]+-[0-9a-f]{8,10}'source_labels:- __meta_kubernetes_pod_controller_name- source_labels:- __meta_kubernetes_pod_controller_nametarget_label: __service__- source_labels:- __meta_kubernetes_pod_node_nametarget_label: __host__- action: dropregex: ''source_labels:- __service__- action: labelmapregex: __meta_kubernetes_pod_label_(.+)- action: replacereplacement: $1separator: /source_labels:- __meta_kubernetes_namespace- __service__target_label: job- action: replacesource_labels:- __meta_kubernetes_namespacetarget_label: namespace- action: replacesource_labels:- __meta_kubernetes_pod_nametarget_label: pod- action: replacesource_labels:- __meta_kubernetes_pod_container_nametarget_label: container- replacement: /var/log/pods/*$1/*.logseparator: /source_labels:- __meta_kubernetes_pod_uid- __meta_kubernetes_pod_container_nametarget_label: __path__- job_name: kubernetes-pods-indirect-controllerpipeline_stages:- docker: {}kubernetes_sd_configs:- role: podrelabel_configs:- action: dropregex: .+separator: ''source_labels:- __meta_kubernetes_pod_label_name- __meta_kubernetes_pod_label_app- action: keepregex: '[0-9a-z-.]+-[0-9a-f]{8,10}'source_labels:- __meta_kubernetes_pod_controller_name- action: replaceregex: '([0-9a-z-.]+)-[0-9a-f]{8,10}'source_labels:- __meta_kubernetes_pod_controller_nametarget_label: __service__- source_labels:- __meta_kubernetes_pod_node_nametarget_label: __host__- action: dropregex: ''source_labels:- __service__- action: labelmapregex: __meta_kubernetes_pod_label_(.+)- action: replacereplacement: $1separator: /source_labels:- __meta_kubernetes_namespace- __service__target_label: job- action: replacesource_labels:- __meta_kubernetes_namespacetarget_label: namespace- action: replacesource_labels:- __meta_kubernetes_pod_nametarget_label: pod- action: replacesource_labels:- __meta_kubernetes_pod_container_nametarget_label: container- replacement: /var/log/pods/*$1/*.logseparator: /source_labels:- __meta_kubernetes_pod_uid- __meta_kubernetes_pod_container_nametarget_label: __path__- job_name: kubernetes-pods-staticpipeline_stages:- docker: {}kubernetes_sd_configs:- role: podrelabel_configs:- action: dropregex: ''source_labels:- __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror- action: replacesource_labels:- __meta_kubernetes_pod_label_componenttarget_label: __service__- source_labels:- __meta_kubernetes_pod_node_nametarget_label: __host__- action: dropregex: ''source_labels:- __service__- action: labelmapregex: __meta_kubernetes_pod_label_(.+)- action: replacereplacement: $1separator: /source_labels:- __meta_kubernetes_namespace- __service__target_label: job- action: replacesource_labels:- __meta_kubernetes_namespacetarget_label: namespace- action: replacesource_labels:- __meta_kubernetes_pod_nametarget_label: pod- action: replacesource_labels:- __meta_kubernetes_pod_container_nametarget_label: container- replacement: /var/log/pods/*$1/*.logseparator: /source_labels:- __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror- __meta_kubernetes_pod_container_nametarget_label: __path__
部署grafana
mkdir -p /data/prometheus/grafana && chmod 755 /data/prometheus/grafana
docker run -d -p 3000:3000 -v /data/prometheus/grafana:/var/lib/grafana --name=grafana grafana/grafana:latest
grafana容器配置文件：/etc/grafana/grafana.ini。
访问ip:3000，初始账号密码为admin/admin，会要求更改密码。
添加数据源loki:
grafana模板：https://grafana.com/grafana/dashboards，我这里使用ID: 13639
