fix: intrument os.OpenFile differently for Reads and Writes (#15449)

allows us to trace latency for READs or WRITEs

Harshavardhana committed 4 years ago

Commit 043aaa71 parent aad9cb28 files · +43 −36Browse files

Unified · Split

8 files

cmd/common-main.go +3 −3
cmd/data-update-tracker.go +1 −1
cmd/disk-cache-backend.go +10 −10
cmd/os-instrumented.go +8 −2
cmd/osmetric_string.go +15 −14
cmd/tier-journal.go +2 −2
cmd/update.go +1 −1
cmd/xl-storage.go +3 −3

@@ -535,7 +535,7 @@ func parsEnvEntry(envEntry string) (envKV, error) { · 534 unmodified lines
535	// the environment values from a file, in the form "key, value".	535	// the environment values from a file, in the form "key, value".
536	// in a structured form.	536	// in a structured form.
537	func minioEnvironFromFile(envConfigFile string) ([]envKV, error) {	537	func minioEnvironFromFile(envConfigFile string) ([]envKV, error) {
538	f, err := os.Open(envConfigFile)	538	f, err := Open(envConfigFile)
539	if err != nil {	539	if err != nil {
540	return nil, err	540	return nil, err
541	}	541	}
@@ -916,7 +916,7 @@ func getTLSConfig() (x509Certs []x509.Certificate, manager certs.Manager, secu · 374 unmodified lines
916	// Therefore, we read all filenames in the cert directory and check	916	// Therefore, we read all filenames in the cert directory and check
917	// for each directory whether it contains a public.crt and private.key.	917	// for each directory whether it contains a public.crt and private.key.
918	// If so, we try to add it to certificate manager.	918	// If so, we try to add it to certificate manager.
919	root, err := os.Open(globalCertsDir.Get())	919	root, err := Open(globalCertsDir.Get())
920	if err != nil {	920	if err != nil {
921	return nil, nil, false, err	921	return nil, nil, false, err
922	}	922	}
@@ -935,7 +935,7 @@ func getTLSConfig() (x509Certs []x509.Certificate, manager certs.Manager, secu · 12 unmodified lines
935	continue	935	continue
936	}	936	}
937	if file.Mode()&os.ModeSymlink == os.ModeSymlink {	937	if file.Mode()&os.ModeSymlink == os.ModeSymlink {
938	file, err = os.Stat(filepath.Join(root.Name(), file.Name()))	938	file, err = Stat(filepath.Join(root.Name(), file.Name()))
939	if err != nil {	939	if err != nil {
940	// not accessible ignore	940	// not accessible ignore
941	continue	941	continue

cmd/data-update-tracker.go+1−1

@@ -227,7 +227,7 @@ func (d *dataUpdateTracker) load(ctx context.Context, drives ...string) { · 226 unmodified lines
227	for _, drive := range drives {	227	for _, drive := range drives {
228		228
229	cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)	229	cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
230	f, err := os.Open(cacheFormatPath)	230	f, err := OpenFile(cacheFormatPath, readMode, 0o666)
231	if err != nil {	231	if err != nil {
232	if osIsNotExist(err) {	232	if osIsNotExist(err) {
233	continue	233	continue

cmd/disk-cache-backend.go+10−10

@@ -624,7 +624,7 @@ func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, met · 623 unmodified lines
624	if err := os.MkdirAll(cachedPath, 0o777); err != nil {	624	if err := os.MkdirAll(cachedPath, 0o777); err != nil {
625	return err	625	return err
626	}	626	}
627	f, err := os.OpenFile(metaPath, os.O_RDWR\|os.O_CREATE, 0o666)	627	f, err := OpenFile(metaPath, os.O_RDWR\|os.O_CREATE\|writeMode, 0o666)
628	if err != nil {	628	if err != nil {
629	return err	629	return err
630	}	630	}
@@ -687,7 +687,7 @@ func (c *diskCache) updateMetadata(ctx context.Context, bucket, object, etag str · 56 unmodified lines
687	if err := os.MkdirAll(cachedPath, 0o777); err != nil {	687	if err := os.MkdirAll(cachedPath, 0o777); err != nil {
688	return err	688	return err
689	}	689	}
690	f, err := os.OpenFile(metaPath, os.O_RDWR, 0o666)	690	f, err := OpenFile(metaPath, os.O_RDWR\|writeMode, 0o666)
691	if err != nil {	691	if err != nil {
692	return err	692	return err
693	}	693	}
@@ -1275,12 +1275,12 @@ func (c *diskCache) NewMultipartUpload(ctx context.Context, bucket, object, uID · 581 unmodified lines
1275		1275
1276	cachePath := getMultipartCacheSHADir(c.dir, bucket, object)	1276	cachePath := getMultipartCacheSHADir(c.dir, bucket, object)
1277	uploadIDDir := path.Join(cachePath, uploadID)	1277	uploadIDDir := path.Join(cachePath, uploadID)
1278	if err := os.MkdirAll(uploadIDDir, 0o777); err != nil {	1278	if err := mkdirAll(uploadIDDir, 0o777); err != nil {
1279	return uploadID, err	1279	return uploadID, err
1280	}	1280	}
1281	metaPath := pathJoin(uploadIDDir, cacheMetaJSONFile)	1281	metaPath := pathJoin(uploadIDDir, cacheMetaJSONFile)
1282		1282
1283	f, err := os.OpenFile(metaPath, os.O_RDWR\|os.O_CREATE, 0o666)	1283	f, err := OpenFile(metaPath, os.O_RDWR\|os.O_CREATE\|writeMode, 0o666)
1284	if err != nil {	1284	if err != nil {
1285	return uploadID, err	1285	return uploadID, err
1286	}	1286	}
@@ -1386,7 +1386,7 @@ func (c *diskCache) SavePartMetadata(ctx context.Context, bucket, object, upload · 99 unmodified lines
1386	defer uploadLock.Unlock(ulkctx.Cancel)	1386	defer uploadLock.Unlock(ulkctx.Cancel)
1387		1387
1388	metaPath := pathJoin(uploadDir, cacheMetaJSONFile)	1388	metaPath := pathJoin(uploadDir, cacheMetaJSONFile)
1389	f, err := os.OpenFile(metaPath, os.O_RDWR, 0o666)	1389	f, err := OpenFile(metaPath, os.O_RDWR\|writeMode, 0o666)
1390	if err != nil {	1390	if err != nil {
1391	return err	1391	return err
1392	}	1392	}
@@ -1465,7 +1465,7 @@ func newCachePartEncryptReader(ctx context.Context, bucket, object string, partI · 72 unmodified lines
1465	func (c *diskCache) uploadIDExists(bucket, object, uploadID string) (err error) {	1465	func (c *diskCache) uploadIDExists(bucket, object, uploadID string) (err error) {
1466	mpartCachePath := getMultipartCacheSHADir(c.dir, bucket, object)	1466	mpartCachePath := getMultipartCacheSHADir(c.dir, bucket, object)
1467	uploadIDDir := path.Join(mpartCachePath, uploadID)	1467	uploadIDDir := path.Join(mpartCachePath, uploadID)
1468	if _, err := os.Stat(uploadIDDir); err != nil {	1468	if _, err := Stat(uploadIDDir); err != nil {
1469	return err	1469	return err
1470	}	1470	}
1471	return nil	1471	return nil
@@ -1564,7 +1564,7 @@ func (c *diskCache) CompleteMultipartUpload(ctx context.Context, bucket, object, · 92 unmodified lines
1564	uploadMeta.Hits++	1564	uploadMeta.Hits++
1565	metaPath := pathJoin(uploadIDDir, cacheMetaJSONFile)	1565	metaPath := pathJoin(uploadIDDir, cacheMetaJSONFile)
1566		1566
1567	f, err := os.OpenFile(metaPath, os.O_RDWR\|os.O_CREATE, 0o666)	1567	f, err := OpenFile(metaPath, os.O_RDWR\|os.O_CREATE\|writeMode, 0o666)
1568	if err != nil {	1568	if err != nil {
1569	return oi, err	1569	return oi, err
1570	}	1570	}
@@ -1634,7 +1634,7 @@ func (c *diskCache) cleanupStaleUploads(ctx context.Context) { · 63 unmodified lines
1634	readDirFn(pathJoin(c.dir, minioMetaBucket, cacheMultipartDir), func(shaDir string, typ os.FileMode) error {	1634	readDirFn(pathJoin(c.dir, minioMetaBucket, cacheMultipartDir), func(shaDir string, typ os.FileMode) error {
1635	return readDirFn(pathJoin(c.dir, minioMetaBucket, cacheMultipartDir, shaDir), func(uploadIDDir string, typ os.FileMode) error {	1635	return readDirFn(pathJoin(c.dir, minioMetaBucket, cacheMultipartDir, shaDir), func(uploadIDDir string, typ os.FileMode) error {
1636	uploadIDPath := pathJoin(c.dir, minioMetaBucket, cacheMultipartDir, shaDir, uploadIDDir)	1636	uploadIDPath := pathJoin(c.dir, minioMetaBucket, cacheMultipartDir, shaDir, uploadIDDir)
1637	fi, err := os.Stat(uploadIDPath)	1637	fi, err := Stat(uploadIDPath)
1638	if err != nil {	1638	if err != nil {
1639	return nil	1639	return nil
1640	}	1640	}
@@ -1649,8 +1649,8 @@ func (c *diskCache) cleanupStaleUploads(ctx context.Context) { · 8 unmodified lines
1649	readDirFn(pathJoin(c.dir, minioMetaBucket, cacheWritebackDir), func(shaDir string, typ os.FileMode) error {	1649	readDirFn(pathJoin(c.dir, minioMetaBucket, cacheWritebackDir), func(shaDir string, typ os.FileMode) error {
1650	wbdir := pathJoin(c.dir, minioMetaBucket, cacheWritebackDir, shaDir)	1650	wbdir := pathJoin(c.dir, minioMetaBucket, cacheWritebackDir, shaDir)
1651	cachedir := pathJoin(c.dir, shaDir)	1651	cachedir := pathJoin(c.dir, shaDir)
1652	if _, err := os.Stat(cachedir); os.IsNotExist(err) {	1652	if _, err := Stat(cachedir); os.IsNotExist(err) {
1653	fi, err := os.Stat(wbdir)	1653	fi, err := Stat(wbdir)
1654	if err != nil {	1654	if err != nil {
1655	return nil	1655	return nil
1656	}	1656	}

cmd/os-instrumented.go+8−2

 	osMetricMkdirAll
 	osMetricMkdir
 	osMetricRename
-	osMetricOpenFile
+	osMetricOpenFileW
+	osMetricOpenFileR
 	osMetricOpen
 	osMetricOpenFileDirectIO
 	osMetricLstat
 // OpenFile captures time taken to call os.OpenFile
 func OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) {
-	defer updateOSMetrics(osMetricOpenFile, name)()
+	switch flag & writeMode {
+	case writeMode:
+		defer updateOSMetrics(osMetricOpenFileW, name)()
+	default:
+		defer updateOSMetrics(osMetricOpenFileR, name)()
+	}
 	return os.OpenFile(name, flag, perm)
 }

cmd/osmetric_string.go+15−14

@@ -12,23 +12,24 @@ func _() { · 11 unmodified lines
12	_ = x[osMetricMkdirAll-1]	12	_ = x[osMetricMkdirAll-1]
13	_ = x[osMetricMkdir-2]	13	_ = x[osMetricMkdir-2]
14	_ = x[osMetricRename-3]	14	_ = x[osMetricRename-3]
15	_ = x[osMetricOpenFile-4]	15	_ = x[osMetricOpenFileW-4]
16	_ = x[osMetricOpen-5]	16	_ = x[osMetricOpenFileR-5]
17	_ = x[osMetricOpenFileDirectIO-6]	17	_ = x[osMetricOpen-6]
18	_ = x[osMetricLstat-7]	18	_ = x[osMetricOpenFileDirectIO-7]
19	_ = x[osMetricRemove-8]	19	_ = x[osMetricLstat-8]
20	_ = x[osMetricStat-9]	20	_ = x[osMetricRemove-9]
21	_ = x[osMetricAccess-10]	21	_ = x[osMetricStat-10]
22	_ = x[osMetricCreate-11]	22	_ = x[osMetricAccess-11]
23	_ = x[osMetricReadDirent-12]	23	_ = x[osMetricCreate-12]
24	_ = x[osMetricFdatasync-13]	24	_ = x[osMetricReadDirent-13]
25	_ = x[osMetricSync-14]	25	_ = x[osMetricFdatasync-14]
26	_ = x[osMetricLast-15]	26	_ = x[osMetricSync-15]
		27	_ = x[osMetricLast-16]
27	}	28	}
28		29
29	const _osMetric_name = "RemoveAllMkdirAllMkdirRenameOpenFileOpenOpenFileDirectIOLstatRemoveStatAccessCreateReadDirentFdatasyncSyncLast"	30	const _osMetric_name = "RemoveAllMkdirAllMkdirRenameOpenFileWOpenFileROpenOpenFileDirectIOLstatRemoveStatAccessCreateReadDirentFdatasyncSyncLast"
30		31
31	var _osMetric_index = [...]uint8{0, 9, 17, 22, 28, 36, 40, 56, 61, 67, 71, 77, 83, 93, 102, 106, 110}	32	var _osMetric_index = [...]uint8{0, 9, 17, 22, 28, 37, 46, 50, 66, 71, 77, 81, 87, 93, 103, 112, 116, 120}
32		33
33	func (i osMetric) String() string {	34	func (i osMetric) String() string {
34	if i >= osMetric(len(_osMetric_index)-1) {	35	if i >= osMetric(len(_osMetric_index)-1) {

cmd/tier-journal.go+2−2

@@ -237,7 +237,7 @@ func (jd *tierDiskJournal) Open() error { · 236 unmodified lines
237	}	237	}
238		238
239	var err error	239	var err error
240	jd.file, err = os.OpenFile(jd.JournalPath(), os.O_APPEND\|os.O_CREATE\|os.O_WRONLY\|writeMode, 0o666)	240	jd.file, err = OpenFile(jd.JournalPath(), os.O_APPEND\|os.O_CREATE\|os.O_WRONLY\|writeMode, 0o666)
241	if err != nil {	241	if err != nil {
242	return err	242	return err
243	}	243	}
@@ -259,7 +259,7 @@ func (jd *tierDiskJournal) Open() error { · 15 unmodified lines
259	}	259	}
260		260
261	func (jd *tierDiskJournal) OpenRO() (io.ReadCloser, error) {	261	func (jd *tierDiskJournal) OpenRO() (io.ReadCloser, error) {
262	file, err := os.Open(jd.ReadOnlyPath())	262	file, err := Open(jd.ReadOnlyPath())
263	if err != nil {	263	if err != nil {
264	return nil, err	264	return nil, err
265	}	265	}

cmd/update.go+1−1

@@ -181,7 +181,7 @@ func IsBOSH() bool { · 180 unmodified lines
181	// Check if this is Helm package installation and report helm chart version	181	// Check if this is Helm package installation and report helm chart version
182	func getHelmVersion(helmInfoFilePath string) string {	182	func getHelmVersion(helmInfoFilePath string) string {
183	// Read the file exists.	183	// Read the file exists.
184	helmInfoFile, err := os.Open(helmInfoFilePath)	184	helmInfoFile, err := Open(helmInfoFilePath)
185	if err != nil {	185	if err != nil {
186	// Log errors and return "" as MinIO can be deployed	186	// Log errors and return "" as MinIO can be deployed
187	// without Helm charts as well.	187	// without Helm charts as well.

cmd/xl-storage.go+3−3

@@ -394,7 +394,7 @@ func (s *xlStorage) readMetadataWithDMTime(ctx context.Context, itemPath string) · 393 unmodified lines
394	return nil, time.Time{}, err	394	return nil, time.Time{}, err
395	}	395	}
396		396
397	f, err := OpenFile(itemPath, readMode, 0)	397	f, err := OpenFile(itemPath, readMode, 0o666)
398	if err != nil {	398	if err != nil {
399	return nil, time.Time{}, err	399	return nil, time.Time{}, err
400	}	400	}
@@ -1547,7 +1547,7 @@ func (s *xlStorage) ReadFile(ctx context.Context, volume string, path string, of · 1146 unmodified lines
1547	}	1547	}
1548		1548
1549	// Open the file for reading.	1549	// Open the file for reading.
1550	file, err := Open(filePath)	1550	file, err := OpenFile(filePath, readMode, 0o666)
1551	if err != nil {	1551	if err != nil {
1552	switch {	1552	switch {
1553	case osIsNotExist(err):	1553	case osIsNotExist(err):
@@ -2515,7 +2515,7 @@ func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolum · 961 unmodified lines
2515		2515
2516	func (s *xlStorage) bitrotVerify(ctx context.Context, partPath string, partSize int64, algo BitrotAlgorithm, sum []byte, shardSize int64) error {	2516	func (s *xlStorage) bitrotVerify(ctx context.Context, partPath string, partSize int64, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
2517	// Open the file for reading.	2517	// Open the file for reading.
2518	file, err := Open(partPath)	2518	file, err := OpenFile(partPath, readMode, 0o666)
2519	if err != nil {	2519	if err != nil {
2520	return osErrToFileErr(err)	2520	return osErrToFileErr(err)
2521	}	2521	}

fix: intrument os.OpenFile differently for Reads and Writes (#15449)

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code