classificador_de_arvores7.py

# -*- coding: utf-8 -*-
"""Classificador de Arvores7.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1wZheJFjKiqh9ZzLdiamrVYvHNm7SSexB

# **0-NOTEBOOK CONFIG**

INICIO DO EM CASO DE ERRO
"""

!pip install --quiet --pre torchvision -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html

import torch
import fastai
!pip install fastai==2.0.15
!pip install fastai2==0.0.30
!pip install fastcore==1.0.16

"""FIM DO EM CASO DE ERRO

CONFIG NORMAL
"""

!pip install fastai --upgrade -q
from fastai.vision.all import *
from fastai.vision.widgets import *

"""# **1-DATASET**



"""

from google.colab import drive
drive.mount('/content/gdrive', force_remount=True)
dir = 'gdrive/My Drive/Dataset/OK'
path=Path(dir)

from google.colab import drive
drive.mount('/content/gdrive', force_remount=True)
root_dir = 'gdrive/My Drive/'
global dir
dir = root_dir + 'Dataset/OK'
path=Path(dir)

!ls "/content/gdrive/My Drive/Dataset/OK"
!ls "/content/gdrive/My Drive/Dataset/Teste"

listy = path.ls()
total = 0
for i in listy:
  if os.path.isdir(i):
    print("there are {0} files in {1}!".format(len(os.listdir(i)),i))
    total +=len(os.listdir(i))
print("there are {} files in total!".format(total))

os.listdir(path)

from PIL import Image
import os
import progressbar
global dir

def thingy(path): 
    if path == "":
      path = dir + path
    listy = os.listdir(path)
    for file in progressbar.progressbar(listy):
        if os.path.isdir(path+"/"+file):
            thingy(path+"/"+file)
        elif not file.split(".")[1] == "jpg":
            Image.open(path + "/" + file).convert('RGB').save("{0}/{1}.jpg".format(path,file.split(".")[0]))

thingy("")

get_files(path)

data = None #limpar var pra rodar de novo
data = get_image_files(path)
data[164]

splitter = RandomSplitter(valid_pct=0.2, seed=23) #altere a semente ao rodar novamente

splitter(data) #133 treino 33 validação

item_tfms = [Resize(456, method='crop')]
batch_tfms=[*aug_transforms(size=224), Normalize.from_stats(*imagenet_stats)] #,,Dihedral(p=1)

blocks=(ImageBlock, CategoryBlock)

get_image_files(path)

trees = DataBlock(blocks=blocks, 
                 get_items=get_image_files, #equivalente ao get_x
                 splitter=splitter,
                 get_y=parent_label,
                 item_tfms=item_tfms,
                 batch_tfms=batch_tfms)

trees.summary(path, show_batch=True)

dls = trees.dataloaders(path)

dls.vocab

"""Mostrar os resultados de diversos transforms da mesma imagem"""

dls.train.show_batch(max_n=7, nrows=1, unique=True)

dls.show_batch(max_n=5)

"""---
# **2-TREINAMENTO DO MODELO - TRANSFER LEARNING**




"""

from fastai.metrics import error_rate

from fastai.callback import *

defaults.callbacks[1] = Recorder(train_metrics=True)
defaults.callbacks

cbs=[ShowGraphCallback,ActivationStats(with_hist=True),SaveModelCallback]

learn = None #clear na variavel

learn = cnn_learner(dls,resnet18,metrics=(error_rate, accuracy),cbs=cbs) #resnet34

"""fastai.vision.models)
you get

['BasicBlock', 'Darknet', 'DynamicUnet', 'ResLayer', 'ResNet', 'SqueezeNet',
 'UnetBlock', 'WideResNet', 'XResNet',
 'alexnet', 'darknet', 'densenet121', 'densenet161', 'densenet169',
 'densenet201', 'resnet101', 'resnet152', 'resnet18', 'resnet34', 'resnet50',
 'squeezenet1_0', 'squeezenet1_1', 'unet', 'vgg16_bn', 'vgg19_bn', 'wrn',
 'wrn_22', 'xception', 'xresnet', 'xresnet101', 'xresnet152', 'xresnet18',
 'xresnet34', 'xresnet50']

fastai.vision.models)
you get

['BasicBlock', 'Darknet', 'DynamicUnet', 'ResLayer', 'ResNet', 'SqueezeNet',
 'UnetBlock', 'WideResNet', 'XResNet',
 'alexnet', 'darknet', 'densenet121', 'densenet161', 'densenet169',
 'densenet201', 'resnet101', 'resnet152', 'resnet18', 'resnet34', 'resnet50',
 'squeezenet1_0', 'squeezenet1_1', 'unet', 'vgg16_bn', 'vgg19_bn', 'wrn',
 'wrn_22', 'xception', 'xresnet', 'xresnet101', 'xresnet152', 'xresnet18',
 'xresnet34', 'xresnet50']
"""

learn.show_training_loop()

list(learn.cbs)

"""##2.1 - TREINO PADRAO FIT_ONE_CYCLE"""

learn.fit_one_cycle(30,1e-2)

"""fastai.vision.models)
you get

['BasicBlock', 'Darknet', 'DynamicUnet', 'ResLayer', 'ResNet', 'SqueezeNet',
 'UnetBlock', 'WideResNet', 'XResNet',
 'alexnet', 'darknet', 'densenet121', 'densenet161', 'densenet169',
 'densenet201', 'resnet101', 'resnet152', 'resnet18', 'resnet34', 'resnet50',
 'squeezenet1_0', 'squeezenet1_1', 'unet', 'vgg16_bn', 'vgg19_bn', 'wrn',
 'wrn_22', 'xception', 'xresnet', 'xresnet101', 'xresnet152', 'xresnet18',
 'xresnet34', 'xresnet50']
"""

learn.save('model1')

"""## 2.2 - INICIO TREINO OTIMIZADO (FINE TUNNING) - FIT_ONE_CYCLE + SLICE DE FAIXA"""

learn.load('model1')

learn.lr_find()

learn.recorder.plot_lr_find()

learn.save('model1')

learn.unfreeze()

max_lr=slice(1e-4, 1e-3)

learn.fit_one_cycle(15,max_lr)

"""FIM DO ESTAGIO DE OTIMIZAÇÃO"""

learn.save('model1')

"""##2.3 - AVALIAÇÃO DO MODELO - TRANSFER LEARNING"""

dls.vocab

learn.load('model1')

interp = ClassificationInterpretation.from_learner(learn)
interp.plot_confusion_matrix() #33 da validacao

learn.show_results()

interp.most_confused(min_val=1)

interp.plot_top_losses(4, nrows=2, figsize = (25,5))

learn.save('model1')

"""
# **3- TESTE - ANALISE DO TESTE**"""

from fastai.vision import *

learn.load('model1')

list = None
list2 = None

import os
x = [os.path.join(r,file) for r,d,f in os.walk("/content/gdrive/My Drive/Dataset/Teste") for file in f]
print(x)
len(x)
list = x[0:len(x)]
len(list)
list2 = sorted(list,reverse=True)
len(list2)

pred_class = []
for x in range (0,len(list2)):
  print(x, 'OK')
  pred_class.append(learn.predict(list2[x]))

print(pred_class[1])
print(pred_class[2])
print(pred_class[18])
print(pred_class[19])
print(pred_class[20])
print(pred_class[21])
print(list2[1])
print(list2[2])
print(list2[18])
print(list2[19])
print(list2[20])
print(list2[21])

list2[4]

pred_class[4][0]

folder = []
for x in range (0,len(list2)):
  print(x,'OK')
  folder.append(os.path.basename(os.path.dirname(list2[x])))

folder[4]

if folder[4] == pred_class[4][0]:
  print('acertou')  
else:
  print('diferente')

cor = 0
err = 0
for x in range(0,len(list2)):
  if folder[x] == pred_class[x][0]:
    print('acertou')
    cor +=1
  else:
    print('errou')
    err +=1
print(cor,err)

acc = (cor/len(list2))*100
print(acc)

predlst = []
labelst = []
for x in range (0,len(list2)):
  predlst.append(pred_class[x][0])
  labelst.append(folder[x])

from sklearn.metrics import accuracy_score
print(accuracy_score(labelst,predlst))

from sklearn.metrics import confusion_matrix
cm = confusion_matrix(labelst, predlst)
cm

import matplotlib.pyplot as plt
plt.matshow(cm)
plt.title('Confusion matrix')
plt.colorbar()
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.show()

learn.export()

list2

folder

pred_class